compute region values using SCCs not iterative flow

The strategy is this:

- we compute SCCs once all outlives constraints are known
- we allocate a set of values **per region** for storing liveness
- we allocate a set of values **per SCC** for storing the final values
- when we add a liveness constraint to the region R, we also add it
  to the final value of the SCC to which R belongs
- then we can apply the constraints by just walking the DAG for the
  SCCs and union'ing the children (which have their liveness
  constraints within)

There are a few intermediate refactorings that I really ought to have
broken out into their own commits:

- reverse the constraint graph so that `R1: R2` means `R1 -> R2` and
  not `R2 -> R1`. This fits better with the SCC computation and new
  style of inference (`->` now means "take value from" and not "push
  value into")
  - this does affect some of the UI tests, since they traverse the
    graph, but mostly the artificial ones and they don't necessarily
    seem worse
- put some things (constraint set, etc) into `Rc`. This lets us root
  them to permit mutation and iteration. It also guarantees they don't
  change, which is critical to the correctness of the algorithm.
- Generalize various helpers that previously operated only on points
  to work on any sort of region element.

src/librustc_data_structures/graph/scc/mod.rs

@@ -54,6 +54,11 @@ impl<N: Idx, S: Idx> Sccs<N, S> {
         self.scc_data.len()
     }
 
+    /// Returns the number of SCCs in the graph.
+    pub fn all_sccs(&self) -> impl Iterator<Item = S> {
+        (0 .. self.scc_data.len()).map(S::new)
+    }
+
     /// Returns the SCC to which a node `r` belongs.
     pub fn scc(&self, r: N) -> S {
         self.scc_indices[r]

src/librustc_mir/borrow_check/nll/constraint_generation.rs

@@ -210,7 +210,7 @@ impl<'cx, 'cg, 'gcx, 'tcx> ConstraintGeneration<'cx, 'cg, 'gcx, 'tcx> {
         for (region, location) in liveness_set {
             debug!("generate: {:#?} is live at {:#?}", region, location);
             let region_vid = regioncx.to_region_vid(region);
-            regioncx.add_live_point(region_vid, *location);
+            regioncx.add_live_element(region_vid, *location);
         }
 
         if let Some(all_facts) = all_facts {
@@ -242,7 +242,7 @@ impl<'cx, 'cg, 'gcx, 'tcx> ConstraintGeneration<'cx, 'cg, 'gcx, 'tcx> {
             .tcx
             .for_each_free_region(&live_ty, |live_region| {
                 let vid = live_region.to_region_vid();
-                self.regioncx.add_live_point(vid, location);
+                self.regioncx.add_live_element(vid, location);
             });
     }
 }

src/librustc_mir/borrow_check/nll/constraints/graph.rs

@@ -0,0 +1,134 @@
+// Copyright 2017 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+use borrow_check::nll::constraints::{ConstraintIndex, ConstraintSet};
+use rustc::ty::RegionVid;
+use rustc_data_structures::graph;
+use rustc_data_structures::indexed_vec::IndexVec;
+
+crate struct ConstraintGraph {
+    first_constraints: IndexVec<RegionVid, Option<ConstraintIndex>>,
+    next_constraints: IndexVec<ConstraintIndex, Option<ConstraintIndex>>,
+}
+
+impl ConstraintGraph {
+    /// Create a "dependency graph" where each region constraint `R1:
+    /// R2` is treated as an edge `R1 -> R2`. We use this graph to
+    /// construct SCCs for region inference but also for error
+    /// reporting.
+    crate fn new(set: &ConstraintSet, num_region_vars: usize) -> Self {
+        let mut first_constraints = IndexVec::from_elem_n(None, num_region_vars);
+        let mut next_constraints = IndexVec::from_elem(None, &set.constraints);
+
+        for (idx, constraint) in set.constraints.iter_enumerated().rev() {
+            let mut head = &mut first_constraints[constraint.sup];
+            let mut next = &mut next_constraints[idx];
+            debug_assert!(next.is_none());
+            *next = *head;
+            *head = Some(idx);
+        }
+
+        Self {
+            first_constraints,
+            next_constraints,
+        }
+    }
+
+    /// Given a region `R`, iterate over all constraints `R: R1`.
+    crate fn outgoing_edges(&self, region_sup: RegionVid) -> Edges<'_> {
+        let first = self.first_constraints[region_sup];
+        Edges {
+            graph: self,
+            pointer: first,
+        }
+    }
+}
+
+crate struct Edges<'s> {
+    graph: &'s ConstraintGraph,
+    pointer: Option<ConstraintIndex>,
+}
+
+impl<'s> Iterator for Edges<'s> {
+    type Item = ConstraintIndex;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        if let Some(p) = self.pointer {
+            self.pointer = self.graph.next_constraints[p];
+            Some(p)
+        } else {
+            None
+        }
+    }
+}
+
+crate struct RegionGraph<'s> {
+    set: &'s ConstraintSet,
+    constraint_graph: &'s ConstraintGraph,
+}
+
+impl<'s> RegionGraph<'s> {
+    /// Create a "dependency graph" where each region constraint `R1:
+    /// R2` is treated as an edge `R1 -> R2`. We use this graph to
+    /// construct SCCs for region inference but also for error
+    /// reporting.
+    crate fn new(set: &'s ConstraintSet, constraint_graph: &'s ConstraintGraph) -> Self {
+        Self {
+            set,
+            constraint_graph,
+        }
+    }
+
+    /// Given a region `R`, iterate over all regions `R1` such that
+    /// there exists a constraint `R: R1`.
+    crate fn sub_regions(&self, region_sup: RegionVid) -> Successors<'_> {
+        Successors {
+            set: self.set,
+            edges: self.constraint_graph.outgoing_edges(region_sup),
+        }
+    }
+}
+
+crate struct Successors<'s> {
+    set: &'s ConstraintSet,
+    edges: Edges<'s>,
+}
+
+impl<'s> Iterator for Successors<'s> {
+    type Item = RegionVid;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        self.edges.next().map(|c| self.set[c].sub)
+    }
+}
+
+impl<'s> graph::DirectedGraph for RegionGraph<'s> {
+    type Node = RegionVid;
+}
+
+impl<'s> graph::WithNumNodes for RegionGraph<'s> {
+    fn num_nodes(&self) -> usize {
+        self.constraint_graph.first_constraints.len()
+    }
+}
+
+impl<'s> graph::WithSuccessors for RegionGraph<'s> {
+    fn successors<'graph>(
+        &'graph self,
+        node: Self::Node,
+    ) -> <Self as graph::GraphSuccessors<'graph>>::Iter {
+        self.sub_regions(node)
+    }
+}
+
+impl<'s, 'graph> graph::GraphSuccessors<'graph> for RegionGraph<'s> {
+    type Item = RegionVid;
+    type Iter = Successors<'graph>;
+}

src/librustc_mir/borrow_check/nll/constraints/mod.rs

@@ -9,19 +9,22 @@
 // except according to those terms.
 
 use rustc::ty::RegionVid;
+use rustc_data_structures::graph::scc::Sccs;
 use rustc_data_structures::indexed_vec::{Idx, IndexVec};
 use borrow_check::nll::type_check::Locations;
 
 use std::fmt;
 use std::ops::Deref;
 
+crate mod graph;
+
 #[derive(Clone, Default)]
 crate struct ConstraintSet {
     constraints: IndexVec<ConstraintIndex, OutlivesConstraint>,
 }
 
 impl ConstraintSet {
-    pub fn push(&mut self, constraint: OutlivesConstraint) {
+    crate fn push(&mut self, constraint: OutlivesConstraint) {
         debug!(
             "ConstraintSet::push({:?}: {:?} @ {:?}",
             constraint.sup, constraint.sub, constraint.locations
@@ -32,12 +35,33 @@ impl ConstraintSet {
         }
         self.constraints.push(constraint);
     }
+
+    /// Constructs a graph from the constraint set; the graph makes it
+    /// easy to find the constriants affecting a particular region
+    /// (you should not mutate the set once this graph is
+    /// constructed).
+    crate fn graph(&self, num_region_vars: usize) -> graph::ConstraintGraph {
+        graph::ConstraintGraph::new(self, num_region_vars)
+    }
+
+    /// Compute cycles (SCCs) in the graph of regions. In particular,
+    /// find all regions R1, R2 such that R1: R2 and R2: R1 and group
+    /// them into an SCC, and find the relationships between SCCs.
+    crate fn compute_sccs(
+        &self,
+        constraint_graph: &graph::ConstraintGraph,
+    ) -> Sccs<RegionVid, ConstraintSccIndex> {
+        let region_graph = &graph::RegionGraph::new(self, constraint_graph);
+        Sccs::new(region_graph)
+    }
 }
 
 impl Deref for ConstraintSet {
     type Target = IndexVec<ConstraintIndex, OutlivesConstraint>;
 
-    fn deref(&self) -> &Self::Target { &self.constraints }
+    fn deref(&self) -> &Self::Target {
+        &self.constraints
+    }
 }
 
 #[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
@@ -68,45 +92,4 @@ impl fmt::Debug for OutlivesConstraint {
 
 newtype_index!(ConstraintIndex { DEBUG_FORMAT = "ConstraintIndex({})" });
 
-crate struct ConstraintGraph {
-    first_constraints: IndexVec<RegionVid, Option<ConstraintIndex>>,
-    next_constraints: IndexVec<ConstraintIndex, Option<ConstraintIndex>>,
-}
-
-impl ConstraintGraph {
-    /// Constraint a graph where each region constraint `R1: R2` is
-    /// treated as an edge `R2 -> R1`. This is useful for cheaply
-    /// finding dirty constraints.
-    crate fn new(set: &ConstraintSet, num_region_vars: usize) -> Self {
-        let mut first_constraints = IndexVec::from_elem_n(None, num_region_vars);
-        let mut next_constraints = IndexVec::from_elem(None, &set.constraints);
-
-        for (idx, constraint) in set.constraints.iter_enumerated().rev() {
-            let mut head = &mut first_constraints[constraint.sub];
-            let mut next = &mut next_constraints[idx];
-            debug_assert!(next.is_none());
-            *next = *head;
-            *head = Some(idx);
-        }
-
-        ConstraintGraph { first_constraints, next_constraints }
-    }
-
-    /// Invokes `op` with the index of any constraints of the form
-    /// `region_sup: region_sub`.  These are the constraints that must
-    /// be reprocessed when the value of `R1` changes. If you think of
-    /// each constraint `R1: R2` as an edge `R2 -> R1`, then this
-    /// gives the set of successors to R2.
-    crate fn for_each_dependent(
-        &self,
-        region_sub: RegionVid,
-        mut op: impl FnMut(ConstraintIndex),
-    ) {
-        let mut p = self.first_constraints[region_sub];
-        while let Some(dep_idx) = p {
-            op(dep_idx);
-            p = self.next_constraints[dep_idx];
-        }
-    }
-}
-
+newtype_index!(ConstraintSccIndex { DEBUG_FORMAT = "ConstraintSccIndex({})" });

src/librustc_mir/borrow_check/nll/explain_borrow/find_use.rs

@@ -57,7 +57,7 @@ impl<'cx, 'gcx, 'tcx> UseFinder<'cx, 'gcx, 'tcx> {
 
         queue.push_back(self.start_point);
         while let Some(p) = queue.pop_front() {
-            if !self.regioncx.region_contains_point(self.region_vid, p) {
+            if !self.regioncx.region_contains(self.region_vid, p) {
                 continue;
             }
 

src/librustc_mir/borrow_check/nll/region_infer/error_reporting/mod.rs

@@ -50,18 +50,10 @@ impl fmt::Display for ConstraintCategory {
 
 impl<'tcx> RegionInferenceContext<'tcx> {
     /// Walks the graph of constraints (where `'a: 'b` is considered
-    /// an edge `'b -> 'a`) to find all paths from `from_region` to
+    /// an edge `'a -> 'b`) to find all paths from `from_region` to
     /// `to_region`. The paths are accumulated into the vector
     /// `results`. The paths are stored as a series of
     /// `ConstraintIndex` values -- in other words, a list of *edges*.
-    ///
-    /// # Parameters
-    ///
-    /// - `from_region`
-    /// When reporting an error, it is useful to be able to determine
-    /// which constraints influenced the region being reported as an
-    /// error. This function finds all of the paths from the
-    /// constraint.
     fn find_constraint_paths_between_regions(
         &self,
         from_region: RegionVid,
@@ -97,25 +89,25 @@ impl<'tcx> RegionInferenceContext<'tcx> {
         // Check if we reached the region we were looking for.
         if target_test(current_region) {
             if !stack.is_empty() {
-                assert_eq!(self.constraints[stack[0]].sub, from_region);
+                assert_eq!(self.constraints[stack[0]].sup, from_region);
                 results.push(stack.clone());
             }
             return;
         }
 
-        self.constraint_graph.for_each_dependent(current_region, |constraint| {
-            assert_eq!(self.constraints[constraint].sub, current_region);
+        for constraint in self.constraint_graph.outgoing_edges(current_region) {
+            assert_eq!(self.constraints[constraint].sup, current_region);
             stack.push(constraint);
             self.find_constraint_paths_between_regions_helper(
                 from_region,
-                self.constraints[constraint].sup,
+                self.constraints[constraint].sub,
                 target_test,
                 visited,
                 stack,
                 results,
             );
             stack.pop();
-        });
+        }
     }
 
     /// This function will return true if a constraint is interesting and false if a constraint
@@ -207,7 +199,7 @@ impl<'tcx> RegionInferenceContext<'tcx> {
         }
 
         // Find all paths
-        let constraint_paths = self.find_constraint_paths_between_regions(outlived_fr, |r| r == fr);
+        let constraint_paths = self.find_constraint_paths_between_regions(fr, |r| r == outlived_fr);
         debug!("report_error: constraint_paths={:#?}", constraint_paths);
 
         // Find the shortest such path.
@@ -316,7 +308,7 @@ impl<'tcx> RegionInferenceContext<'tcx> {
 
         while changed {
             changed = false;
-            for constraint in &*self.constraints {
+            for constraint in self.constraints.iter() {
                 if let Some(n) = result_set[constraint.sup] {
                     let m = n + 1;
                     if result_set[constraint.sub]

src/librustc_mir/borrow_check/nll/region_infer/mod.rs

@@ -10,8 +10,10 @@
 
 use super::universal_regions::UniversalRegions;
 use borrow_check::nll::constraints::{
-    ConstraintIndex, ConstraintGraph, ConstraintSet, OutlivesConstraint
+    ConstraintIndex, ConstraintSccIndex, ConstraintSet, OutlivesConstraint,
 };
+use borrow_check::nll::constraints::graph::ConstraintGraph;
+use borrow_check::nll::region_infer::values::ToElementIndex;
 use borrow_check::nll::type_check::Locations;
 use rustc::hir::def_id::DefId;
 use rustc::infer::canonical::QueryRegionConstraint;
@@ -25,8 +27,9 @@ use rustc::mir::{
 };
 use rustc::ty::{self, RegionVid, Ty, TyCtxt, TypeFoldable};
 use rustc::util::common;
-use rustc_data_structures::bitvec::BitVector;
-use rustc_data_structures::indexed_vec::{Idx, IndexVec};
+use rustc_data_structures::graph::scc::Sccs;
+use rustc_data_structures::indexed_set::{IdxSet, IdxSetBuf};
+use rustc_data_structures::indexed_vec::IndexVec;
 
 use std::rc::Rc;
 
@@ -55,22 +58,29 @@ pub struct RegionInferenceContext<'tcx> {
     /// the entire CFG and `end(R)`.
     liveness_constraints: RegionValues<RegionVid>,
 
-    /// The final inferred values of the inference variables; `None`
-    /// until `solve` is invoked.
-    inferred_values: Option<RegionValues<RegionVid>>,
+    /// The outlives constraints computed by the type-check.
+    constraints: Rc<ConstraintSet>,
 
-    /// The constraints we have accumulated and used during solving.
-    constraints: ConstraintSet,
+    /// The constraint-set, but in graph form, making it easy to traverse
+    /// the constraints adjacent to a particular region. Used to construct
+    /// the SCC (see `constraint_sccs`) and for error reporting.
+    constraint_graph: Rc<ConstraintGraph>,
 
-    /// The constraint-set, but organized by regions.
-    constraint_graph: ConstraintGraph,
+    /// The SCC computed from `constraints` and
+    /// `constraint_graph`. Used to compute the values of each region.
+    constraint_sccs: Rc<Sccs<RegionVid, ConstraintSccIndex>>,
+
+    /// The final inferred values of the region variables; we compute
+    /// one value per SCC. To get the value for any given *region*,
+    /// you first find which scc it is a part of.
+    scc_values: RegionValues<ConstraintSccIndex>,
 
     /// Type constraints that we check after solving.
     type_tests: Vec<TypeTest<'tcx>>,
 
     /// Information about the universally quantified regions in scope
     /// on this function and their (known) relations to one another.
-    universal_regions: UniversalRegions<'tcx>,
+    universal_regions: Rc<UniversalRegions<'tcx>>,
 }
 
 struct RegionDefinition<'tcx> {
@@ -201,6 +211,7 @@ impl<'tcx> RegionInferenceContext<'tcx> {
         outlives_constraints: ConstraintSet,
         type_tests: Vec<TypeTest<'tcx>>,
     ) -> Self {
+        let universal_regions = Rc::new(universal_regions);
         let num_region_variables = var_infos.len();
         let num_universal_regions = universal_regions.len();
 
@@ -212,15 +223,20 @@ impl<'tcx> RegionInferenceContext<'tcx> {
             .map(|info| RegionDefinition::new(info.origin))
             .collect();
 
-        let constraint_graph = ConstraintGraph::new(&outlives_constraints, definitions.len());
+        let constraints = Rc::new(outlives_constraints); // freeze constraints
+        let constraint_graph = Rc::new(constraints.graph(definitions.len()));
+        let constraint_sccs = Rc::new(constraints.compute_sccs(&constraint_graph));
+
+        let scc_values = RegionValues::new(elements, constraint_sccs.num_sccs());
 
         let mut result = Self {
             definitions,
             elements: elements.clone(),
             liveness_constraints: RegionValues::new(elements, num_region_variables),
-            inferred_values: None,
-            constraints: outlives_constraints,
+            constraints,
+            constraint_sccs,
             constraint_graph,
+            scc_values,
             type_tests,
             universal_regions,
         };
@@ -262,7 +278,9 @@ impl<'tcx> RegionInferenceContext<'tcx> {
         }
 
         // For each universally quantified region X:
-        for variable in self.universal_regions.universal_regions() {
+        let elements = self.elements.clone();
+        let universal_regions = self.universal_regions.clone();
+        for variable in universal_regions.universal_regions() {
             // These should be free-region variables.
             assert!(match self.definitions[variable].origin {
                 RegionVariableOrigin::NLL(NLLRegionVariableOrigin::FreeRegion) => true,
@@ -272,12 +290,12 @@ impl<'tcx> RegionInferenceContext<'tcx> {
             self.definitions[variable].is_universal = true;
 
             // Add all nodes in the CFG to liveness constraints
-            for point_index in self.elements.all_point_indices() {
-                self.liveness_constraints.add_element(variable, point_index);
+            for point_index in elements.all_point_indices() {
+                self.add_live_element(variable, point_index);
             }
 
             // Add `end(X)` into the set for X.
-            self.liveness_constraints.add_element(variable, variable);
+            self.add_live_element(variable, variable);
         }
     }
 
@@ -297,37 +315,38 @@ impl<'tcx> RegionInferenceContext<'tcx> {
     /// Returns true if the region `r` contains the point `p`.
     ///
     /// Panics if called before `solve()` executes,
-    pub fn region_contains_point<R>(&self, r: R, p: Location) -> bool
-    where
-        R: ToRegionVid,
-    {
-        let inferred_values = self
-            .inferred_values
-            .as_ref()
-            .expect("region values not yet inferred");
-        inferred_values.contains(r.to_region_vid(), p)
+    crate fn region_contains(&self, r: impl ToRegionVid, p: impl ToElementIndex) -> bool {
+        let scc = self.constraint_sccs.scc(r.to_region_vid());
+        self.scc_values.contains(scc, p)
     }
 
     /// Returns access to the value of `r` for debugging purposes.
     crate fn region_value_str(&self, r: RegionVid) -> String {
-        let inferred_values = self
-            .inferred_values
-            .as_ref()
-            .expect("region values not yet inferred");
-
-        inferred_values.region_value_str(r)
+        let scc = self.constraint_sccs.scc(r.to_region_vid());
+        self.scc_values.region_value_str(scc)
     }
 
     /// Indicates that the region variable `v` is live at the point `point`.
     ///
     /// Returns `true` if this constraint is new and `false` is the
     /// constraint was already present.
-    pub(super) fn add_live_point(&mut self, v: RegionVid, point: Location) -> bool {
-        debug!("add_live_point({:?}, {:?})", v, point);
-        assert!(self.inferred_values.is_none(), "values already inferred");
+    pub(super) fn add_live_element(
+        &mut self,
+        v: RegionVid,
+        elem: impl ToElementIndex,
+    ) -> bool {
+        debug!("add_live_element({:?}, {:?})", v, elem);
 
-        let element = self.elements.index(point);
-        self.liveness_constraints.add_element(v, element)
+        // Add to the liveness values for `v`...
+        if self.liveness_constraints.add_element(v, elem) {
+            // ...but also add to the SCC in which `v` appears.
+            let scc = self.constraint_sccs.scc(v);
+            self.scc_values.add_element(scc, elem);
+
+            true
+        } else {
+            false
+        }
     }
 
     /// Perform region inference and report errors if we see any
@@ -352,8 +371,6 @@ impl<'tcx> RegionInferenceContext<'tcx> {
         mir: &Mir<'tcx>,
         mir_def_id: DefId,
     ) -> Option<ClosureRegionRequirements<'gcx>> {
-        assert!(self.inferred_values.is_none(), "values already inferred");
-
         self.propagate_constraints(mir);
 
         // If this is a closure, we can propagate unsatisfied
@@ -388,56 +405,62 @@ impl<'tcx> RegionInferenceContext<'tcx> {
     /// for each region variable until all the constraints are
     /// satisfied. Note that some values may grow **too** large to be
     /// feasible, but we check this later.
-    fn propagate_constraints(&mut self, mir: &Mir<'tcx>) {
-        assert!(self.inferred_values.is_none());
-        let inferred_values = self.compute_region_values(mir);
-        self.inferred_values = Some(inferred_values);
-    }
+    fn propagate_constraints(&mut self, _mir: &Mir<'tcx>) {
+        debug!("propagate_constraints()");
 
-    fn compute_region_values(&self, _mir: &Mir<'tcx>) -> RegionValues<RegionVid> {
-        debug!("compute_region_values()");
-        debug!("compute_region_values: constraints={:#?}", {
+        debug!("propagate_constraints: constraints={:#?}", {
             let mut constraints: Vec<_> = self.constraints.iter().collect();
             constraints.sort();
             constraints
         });
 
-        // The initial values for each region are derived from the liveness
-        // constraints we have accumulated.
-        let mut inferred_values = self.liveness_constraints.clone();
+        // To propagate constriants, we walk the DAG induced by the
+        // SCC. For each SCC, we visit its successors and compute
+        // their values, then we union all those values to get our
+        // own.
+        let visited = &mut IdxSetBuf::new_empty(self.constraint_sccs.num_sccs());
+        for scc_index in self.constraint_sccs.all_sccs() {
+            self.propagate_constraint_sccs_if_new(scc_index, visited);
+        }
+    }
 
-        // Constraints that may need to be repropagated (initially all):
-        let mut dirty_list: Vec<_> = self.constraints.indices().collect();
+    #[inline]
+    fn propagate_constraint_sccs_if_new(
+        &mut self,
+        scc_a: ConstraintSccIndex,
+        visited: &mut IdxSet<ConstraintSccIndex>,
+    ) {
+        if visited.add(&scc_a) {
+            self.propagate_constraint_sccs_new(scc_a, visited);
+        }
+    }
 
-        // Set to 0 for each constraint that is on the dirty list:
-        let mut clean_bit_vec = BitVector::new(dirty_list.len());
+    fn propagate_constraint_sccs_new(
+        &mut self,
+        scc_a: ConstraintSccIndex,
+        visited: &mut IdxSet<ConstraintSccIndex>,
+    ) {
+        let constraint_sccs = self.constraint_sccs.clone();
 
-        debug!("propagate_constraints: --------------------");
-        while let Some(constraint_idx) = dirty_list.pop() {
-            clean_bit_vec.insert(constraint_idx.index());
+        // Walk each SCC `B` such that `A: B`...
+        for &scc_b in constraint_sccs.successors(scc_a) {
+            debug!(
+                "propagate_constraint_sccs: scc_a = {:?} scc_b = {:?}",
+                scc_a, scc_b
+            );
 
-            let constraint = &self.constraints[constraint_idx];
-            debug!("propagate_constraints: constraint={:?}", constraint);
+            // ...compute the value of `B`...
+            self.propagate_constraint_sccs_if_new(scc_b, visited);
 
-            if inferred_values.add_region(constraint.sup, constraint.sub) {
-                debug!("propagate_constraints:   sub={:?}", constraint.sub);
-                debug!("propagate_constraints:   sup={:?}", constraint.sup);
-
-                // The region of `constraint.sup` changed, so find all
-                // constraints of the form `R: constriant.sup` and
-                // enqueue them as dirty.  We will have to reprocess
-                // them.
-                self.constraint_graph.for_each_dependent(constraint.sup, |dep_idx| {
-                    if clean_bit_vec.remove(dep_idx.index()) {
-                        dirty_list.push(dep_idx);
-                    }
-                });
-            }
-
-            debug!("\n");
+            // ...and add elements from `B` into `A`.
+            self.scc_values.add_region(scc_a, scc_b);
         }
 
-        inferred_values
+        debug!(
+            "propagate_constraint_sccs: scc_a = {:?} has value {:?}",
+            scc_a,
+            self.scc_values.region_value_str(scc_a),
+        );
     }
 
     /// Once regions have been propagated, this method is used to see
@@ -512,12 +535,9 @@ impl<'tcx> RegionInferenceContext<'tcx> {
         if self.universal_regions.is_universal_region(r) {
             return self.definitions[r].external_name;
         } else {
-            let inferred_values = self
-                .inferred_values
-                .as_ref()
-                .expect("region values not yet inferred");
+            let r_scc = self.constraint_sccs.scc(r);
             let upper_bound = self.universal_upper_bound(r);
-            if inferred_values.contains(r, upper_bound) {
+            if self.scc_values.contains(r_scc, upper_bound) {
                 self.to_error_region(upper_bound)
             } else {
                 None
@@ -552,11 +572,8 @@ impl<'tcx> RegionInferenceContext<'tcx> {
         // region, which ensures it can be encoded in a `ClosureOutlivesRequirement`.
         let lower_bound_plus = self.non_local_universal_upper_bound(*lower_bound);
         assert!(self.universal_regions.is_universal_region(lower_bound_plus));
-        assert!(
-            !self
-                .universal_regions
-                .is_local_free_region(lower_bound_plus)
-        );
+        assert!(!self.universal_regions
+            .is_local_free_region(lower_bound_plus));
 
         propagated_outlives_requirements.push(ClosureOutlivesRequirement {
             subject,
@@ -584,10 +601,6 @@ impl<'tcx> RegionInferenceContext<'tcx> {
     ) -> Option<ClosureOutlivesSubject<'gcx>> {
         let tcx = infcx.tcx;
         let gcx = tcx.global_tcx();
-        let inferred_values = self
-            .inferred_values
-            .as_ref()
-            .expect("region values not yet inferred");
 
         debug!("try_promote_type_test_subject(ty = {:?})", ty);
 
@@ -630,7 +643,7 @@ impl<'tcx> RegionInferenceContext<'tcx> {
             // `'static` is not contained in `r`, we would fail to
             // find an equivalent.
             let upper_bound = self.non_local_universal_upper_bound(region_vid);
-            if inferred_values.contains(region_vid, upper_bound) {
+            if self.region_contains(region_vid, upper_bound) {
                 tcx.mk_region(ty::ReClosureBound(upper_bound))
             } else {
                 // In the case of a failure, use a `ReVar`
@@ -663,12 +676,10 @@ impl<'tcx> RegionInferenceContext<'tcx> {
     /// except that it converts further takes the non-local upper
     /// bound of `'y`, so that the final result is non-local.
     fn non_local_universal_upper_bound(&self, r: RegionVid) -> RegionVid {
-        let inferred_values = self.inferred_values.as_ref().unwrap();
-
         debug!(
             "non_local_universal_upper_bound(r={:?}={})",
             r,
-            inferred_values.region_value_str(r)
+            self.region_value_str(r)
         );
 
         let lub = self.universal_upper_bound(r);
@@ -700,18 +711,17 @@ impl<'tcx> RegionInferenceContext<'tcx> {
     /// - For each `end('x)` element in `'r`, compute the mutual LUB, yielding
     ///   a result `'y`.
     fn universal_upper_bound(&self, r: RegionVid) -> RegionVid {
-        let inferred_values = self.inferred_values.as_ref().unwrap();
-
         debug!(
             "universal_upper_bound(r={:?}={})",
             r,
-            inferred_values.region_value_str(r)
+            self.region_value_str(r)
         );
 
         // Find the smallest universal region that contains all other
         // universal regions within `region`.
         let mut lub = self.universal_regions.fr_fn_body;
-        for ur in inferred_values.universal_regions_outlived_by(r) {
+        let r_scc = self.constraint_sccs.scc(r);
+        for ur in self.scc_values.universal_regions_outlived_by(r_scc) {
             lub = self.universal_regions.postdom_upper_bound(lub, ur);
         }
 
@@ -756,31 +766,29 @@ impl<'tcx> RegionInferenceContext<'tcx> {
     ) -> bool {
         debug!("eval_outlives({:?}: {:?})", sup_region, sub_region);
 
-        let inferred_values = self
-            .inferred_values
-            .as_ref()
-            .expect("values for regions not yet inferred");
-
         debug!(
             "eval_outlives: sup_region's value = {:?}",
-            inferred_values.region_value_str(sup_region),
+            self.region_value_str(sup_region),
         );
         debug!(
             "eval_outlives: sub_region's value = {:?}",
-            inferred_values.region_value_str(sub_region),
+            self.region_value_str(sub_region),
         );
 
+        let sub_region_scc = self.constraint_sccs.scc(sub_region);
+        let sup_region_scc = self.constraint_sccs.scc(sup_region);
+
         // Both the `sub_region` and `sup_region` consist of the union
         // of some number of universal regions (along with the union
         // of various points in the CFG; ignore those points for
         // now). Therefore, the sup-region outlives the sub-region if,
         // for each universal region R1 in the sub-region, there
         // exists some region R2 in the sup-region that outlives R1.
-        let universal_outlives = inferred_values
-            .universal_regions_outlived_by(sub_region)
+        let universal_outlives = self.scc_values
+            .universal_regions_outlived_by(sub_region_scc)
             .all(|r1| {
-                inferred_values
-                    .universal_regions_outlived_by(sup_region)
+                self.scc_values
+                    .universal_regions_outlived_by(sup_region_scc)
                     .any(|r2| self.universal_regions.outlives(r2, r1))
             });
 
@@ -796,7 +804,8 @@ impl<'tcx> RegionInferenceContext<'tcx> {
             return true;
         }
 
-        inferred_values.contains_points(sup_region, sub_region)
+        self.scc_values
+            .contains_points(sup_region_scc, sub_region_scc)
     }
 
     /// Once regions have been propagated, this method is used to see
@@ -825,8 +834,7 @@ impl<'tcx> RegionInferenceContext<'tcx> {
     ) {
         // The universal regions are always found in a prefix of the
         // full list.
-        let universal_definitions = self
-            .definitions
+        let universal_definitions = self.definitions
             .iter_enumerated()
             .take_while(|(_, fr_definition)| fr_definition.is_universal);
 
@@ -860,13 +868,13 @@ impl<'tcx> RegionInferenceContext<'tcx> {
         longer_fr: RegionVid,
         propagated_outlives_requirements: &mut Option<&mut Vec<ClosureOutlivesRequirement<'gcx>>>,
     ) {
-        let inferred_values = self.inferred_values.as_ref().unwrap();
-
         debug!("check_universal_region(fr={:?})", longer_fr);
 
+        let longer_fr_scc = self.constraint_sccs.scc(longer_fr);
+
         // Find every region `o` such that `fr: o`
         // (because `fr` includes `end(o)`).
-        for shorter_fr in inferred_values.universal_regions_outlived_by(longer_fr) {
+        for shorter_fr in self.scc_values.universal_regions_outlived_by(longer_fr_scc) {
             // If it is known that `fr: o`, carry on.
             if self.universal_regions.outlives(longer_fr, shorter_fr) {
                 continue;

src/librustc_mir/borrow_check/nll/region_infer/values.rs

@@ -18,7 +18,7 @@ use std::rc::Rc;
 
 /// Maps between the various kinds of elements of a region value to
 /// the internal indices that w use.
-pub(super) struct RegionValueElements {
+crate struct RegionValueElements {
     /// For each basic block, how many points are contained within?
     statements_before_block: IndexVec<BasicBlock, usize>,
     num_points: usize,
@@ -26,7 +26,7 @@ pub(super) struct RegionValueElements {
 }
 
 impl RegionValueElements {
-    pub(super) fn new(mir: &Mir<'_>, num_universal_regions: usize) -> Self {
+    crate fn new(mir: &Mir<'_>, num_universal_regions: usize) -> Self {
         let mut num_points = 0;
         let statements_before_block = mir
             .basic_blocks()
@@ -56,22 +56,22 @@ impl RegionValueElements {
     }
 
     /// Total number of element indices that exist.
-    pub(super) fn num_elements(&self) -> usize {
+    crate fn num_elements(&self) -> usize {
         self.num_points + self.num_universal_regions
     }
 
     /// Converts an element of a region value into a `RegionElementIndex`.
-    pub(super) fn index<T: ToElementIndex>(&self, elem: T) -> RegionElementIndex {
+    crate fn index<T: ToElementIndex>(&self, elem: T) -> RegionElementIndex {
         elem.to_element_index(self)
     }
 
     /// Iterates over the `RegionElementIndex` for all points in the CFG.
-    pub(super) fn all_point_indices<'a>(&'a self) -> impl Iterator<Item = RegionElementIndex> + 'a {
+    crate fn all_point_indices<'a>(&'a self) -> impl Iterator<Item = RegionElementIndex> + 'a {
         (0..self.num_points).map(move |i| RegionElementIndex::new(i + self.num_universal_regions))
     }
 
     /// Converts a particular `RegionElementIndex` to the `RegionElement` it represents.
-    pub(super) fn to_element(&self, i: RegionElementIndex) -> RegionElement {
+    crate fn to_element(&self, i: RegionElementIndex) -> RegionElement {
         debug!("to_element(i={:?})", i);
 
         if let Some(r) = self.to_universal_region(i) {
@@ -114,7 +114,7 @@ impl RegionValueElements {
     /// Converts a particular `RegionElementIndex` to a universal
     /// region, if that is what it represents. Returns `None`
     /// otherwise.
-    pub(super) fn to_universal_region(&self, i: RegionElementIndex) -> Option<RegionVid> {
+    crate fn to_universal_region(&self, i: RegionElementIndex) -> Option<RegionVid> {
         if i.index() < self.num_universal_regions {
             Some(RegionVid::new(i.index()))
         } else {
@@ -138,7 +138,7 @@ newtype_index!(RegionElementIndex { DEBUG_FORMAT = "RegionElementIndex({})" });
 /// An individual element in a region value -- the value of a
 /// particular region variable consists of a set of these elements.
 #[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)]
-pub(super) enum RegionElement {
+crate enum RegionElement {
     /// A point in the control-flow graph.
     Location(Location),
 
@@ -146,7 +146,7 @@ pub(super) enum RegionElement {
     UniversalRegion(RegionVid),
 }
 
-pub(super) trait ToElementIndex: Debug + Copy {
+crate trait ToElementIndex: Debug + Copy {
     fn to_element_index(self, elements: &RegionValueElements) -> RegionElementIndex;
 }
 
@@ -179,7 +179,7 @@ impl ToElementIndex for RegionElementIndex {
 /// variable. The columns consist of either universal regions or
 /// points in the CFG.
 #[derive(Clone)]
-pub(super) struct RegionValues<N: Idx> {
+crate struct RegionValues<N: Idx> {
     elements: Rc<RegionValueElements>,
     matrix: SparseBitMatrix<N, RegionElementIndex>,
 }
@@ -188,7 +188,7 @@ impl<N: Idx> RegionValues<N> {
     /// Creates a new set of "region values" that tracks causal information.
     /// Each of the regions in num_region_variables will be initialized with an
     /// empty set of points and no causal information.
-    pub(super) fn new(elements: &Rc<RegionValueElements>, num_region_variables: usize) -> Self {
+    crate fn new(elements: &Rc<RegionValueElements>, num_region_variables: usize) -> Self {
         assert!(
             elements.num_universal_regions <= num_region_variables,
             "universal regions are a subset of the region variables"
@@ -205,7 +205,11 @@ impl<N: Idx> RegionValues<N> {
 
     /// Adds the given element to the value for the given region. Returns true if
     /// the element is newly added (i.e., was not already present).
-    pub(super) fn add_element<E: ToElementIndex>(&mut self, r: N, elem: E) -> bool {
+    crate fn add_element(
+        &mut self,
+        r: N,
+        elem: impl ToElementIndex,
+    ) -> bool {
         let i = self.elements.index(elem);
         debug!("add(r={:?}, elem={:?})", r, elem);
         self.matrix.add(r, i)
@@ -213,19 +217,19 @@ impl<N: Idx> RegionValues<N> {
 
     /// Add all elements in `r_from` to `r_to` (because e.g. `r_to:
     /// r_from`).
-    pub(super) fn add_region(&mut self, r_to: N, r_from: N) -> bool {
+    crate fn add_region(&mut self, r_to: N, r_from: N) -> bool {
         self.matrix.merge(r_from, r_to)
     }
 
     /// True if the region `r` contains the given element.
-    pub(super) fn contains<E: ToElementIndex>(&self, r: N, elem: E) -> bool {
+    crate fn contains(&self, r: N, elem: impl ToElementIndex) -> bool {
         let i = self.elements.index(elem);
         self.matrix.contains(r, i)
     }
 
     /// True if `sup_region` contains all the CFG points that
     /// `sub_region` contains. Ignores universal regions.
-    pub(super) fn contains_points(&self, sup_region: N, sub_region: N) -> bool {
+    crate fn contains_points(&self, sup_region: N, sub_region: N) -> bool {
         // This could be done faster by comparing the bitsets. But I
         // am lazy.
         self.element_indices_contained_in(sub_region)
@@ -236,7 +240,7 @@ impl<N: Idx> RegionValues<N> {
     /// Iterate over the value of the region `r`, yielding up element
     /// indices. You may prefer `universal_regions_outlived_by` or
     /// `elements_contained_in`.
-    pub(super) fn element_indices_contained_in<'a>(
+    crate fn element_indices_contained_in<'a>(
         &'a self,
         r: N,
     ) -> impl Iterator<Item = RegionElementIndex> + 'a {
@@ -244,7 +248,7 @@ impl<N: Idx> RegionValues<N> {
     }
 
     /// Returns just the universal regions that are contained in a given region's value.
-    pub(super) fn universal_regions_outlived_by<'a>(
+    crate fn universal_regions_outlived_by<'a>(
         &'a self,
         r: N,
     ) -> impl Iterator<Item = RegionVid> + 'a {
@@ -255,7 +259,7 @@ impl<N: Idx> RegionValues<N> {
     }
 
     /// Returns all the elements contained in a given region's value.
-    pub(super) fn elements_contained_in<'a>(
+    crate fn elements_contained_in<'a>(
         &'a self,
         r: N,
     ) -> impl Iterator<Item = RegionElement> + 'a {
@@ -264,7 +268,7 @@ impl<N: Idx> RegionValues<N> {
     }
 
     /// Returns a "pretty" string value of the region. Meant for debugging.
-    pub(super) fn region_value_str(&self, r: N) -> String {
+    crate fn region_value_str(&self, r: N) -> String {
         let mut result = String::new();
         result.push_str("{");
 

src/librustc_mir/dataflow/impls/borrows.rs

@@ -76,7 +76,7 @@ fn precompute_borrows_out_of_scope<'a, 'tcx>(
     while let Some(location) = stack.pop() {
         // If region does not contain a point at the location, then add to list and skip
         // successor locations.
-        if !regioncx.region_contains_point(borrow_region, location) {
+        if !regioncx.region_contains(borrow_region, location) {
             debug!("borrow {:?} gets killed at {:?}", borrow_index, location);
             borrows_out_of_scope_at_location
                 .entry(location)

src/test/ui/nll/closure-requirements/propagate-approximated-fail-no-postdom.rs

@@ -52,9 +52,9 @@ fn supply<'a, 'b, 'c>(cell_a: Cell<&'a u32>, cell_b: Cell<&'b u32>, cell_c: Cell
         cell_c,
         |_outlives1, _outlives2, _outlives3, x, y| {
             // Only works if 'x: 'y:
-            let p = x.get();
+            let p = x.get(); //~ ERROR
             //~^ WARN not reporting region error due to nll
-            demand_y(x, y, p) //~ ERROR
+            demand_y(x, y, p)
         },
     );
 }

src/test/ui/nll/closure-requirements/propagate-approximated-fail-no-postdom.stderr

@@ -1,28 +1,28 @@
 warning: not reporting region error due to nll
   --> $DIR/propagate-approximated-fail-no-postdom.rs:55:21
    |
-LL |             let p = x.get();
+LL |             let p = x.get(); //~ ERROR
    |                     ^^^^^^^
 
 error: unsatisfied lifetime constraints
-  --> $DIR/propagate-approximated-fail-no-postdom.rs:57:13
+  --> $DIR/propagate-approximated-fail-no-postdom.rs:55:21
    |
 LL |         |_outlives1, _outlives2, _outlives3, x, y| {
    |          ----------              ---------- lifetime `'2` appears in this argument
    |          |
    |          lifetime `'1` appears in this argument
-...
-LL |             demand_y(x, y, p) //~ ERROR
-   |             ^^^^^^^^^^^^^^^^^ argument requires that `'1` must outlive `'2`
+LL |             // Only works if 'x: 'y:
+LL |             let p = x.get(); //~ ERROR
+   |                     ^^^^^^^ argument requires that `'1` must outlive `'2`
 
 note: No external requirements
   --> $DIR/propagate-approximated-fail-no-postdom.rs:53:9
    |
 LL | /         |_outlives1, _outlives2, _outlives3, x, y| {
 LL | |             // Only works if 'x: 'y:
-LL | |             let p = x.get();
+LL | |             let p = x.get(); //~ ERROR
 LL | |             //~^ WARN not reporting region error due to nll
-LL | |             demand_y(x, y, p) //~ ERROR
+LL | |             demand_y(x, y, p)
 LL | |         },
    | |_________^
    |

src/test/ui/nll/closure-requirements/propagate-approximated-shorter-to-static-comparing-against-free.stderr

@@ -5,7 +5,7 @@ LL |     foo(cell, |cell_a, cell_x| {
    |     ^^^
 
 error: unsatisfied lifetime constraints
-  --> $DIR/propagate-approximated-shorter-to-static-comparing-against-free.rs:33:9
+  --> $DIR/propagate-approximated-shorter-to-static-comparing-against-free.rs:33:20
    |
 LL |     foo(cell, |cell_a, cell_x| {
    |                ------  ------ lifetime `'1` appears in this argument
@@ -13,7 +13,7 @@ LL |     foo(cell, |cell_a, cell_x| {
    |                lifetime `'2` appears in this argument
 LL |         //~^ WARNING not reporting region error due to nll
 LL |         cell_a.set(cell_x.get()); // forces 'x: 'a, error in closure
-   |         ^^^^^^^^^^^^^^^^^^^^^^^^ argument requires that `'1` must outlive `'2`
+   |                    ^^^^^^^^^^^^ argument requires that `'1` must outlive `'2`
 
 note: No external requirements
   --> $DIR/propagate-approximated-shorter-to-static-comparing-against-free.rs:31:15

src/test/ui/nll/closure-requirements/propagate-fail-to-approximate-longer-no-bounds.stderr

@@ -5,7 +5,7 @@ LL |         demand_y(x, y, x.get())
    |         ^^^^^^^^^^^^^^^^^^^^^^^
 
 error: unsatisfied lifetime constraints
-  --> $DIR/propagate-fail-to-approximate-longer-no-bounds.rs:47:9
+  --> $DIR/propagate-fail-to-approximate-longer-no-bounds.rs:47:24
    |
 LL |     establish_relationships(&cell_a, &cell_b, |_outlives, x, y| {
    |                                                ---------  - lifetime `'1` appears in this argument
@@ -13,7 +13,7 @@ LL |     establish_relationships(&cell_a, &cell_b, |_outlives, x, y| {
    |                                                lifetime `'2` appears in this argument
 LL |         // Only works if 'x: 'y:
 LL |         demand_y(x, y, x.get())
-   |         ^^^^^^^^^^^^^^^^^^^^^^^ argument requires that `'1` must outlive `'2`
+   |                        ^^^^^^^ argument requires that `'1` must outlive `'2`
 
 note: No external requirements
   --> $DIR/propagate-fail-to-approximate-longer-no-bounds.rs:45:47

src/test/ui/nll/closure-requirements/propagate-fail-to-approximate-longer-wrong-bounds.stderr

@@ -5,7 +5,7 @@ LL |         demand_y(x, y, x.get())
    |         ^^^^^^^^^^^^^^^^^^^^^^^
 
 error: unsatisfied lifetime constraints
-  --> $DIR/propagate-fail-to-approximate-longer-wrong-bounds.rs:51:9
+  --> $DIR/propagate-fail-to-approximate-longer-wrong-bounds.rs:51:24
    |
 LL |     establish_relationships(&cell_a, &cell_b, |_outlives1, _outlives2, x, y| {
    |                                                ----------  ---------- lifetime `'2` appears in this argument
@@ -13,7 +13,7 @@ LL |     establish_relationships(&cell_a, &cell_b, |_outlives1, _outlives2, x, y
    |                                                lifetime `'1` appears in this argument
 LL |         // Only works if 'x: 'y:
 LL |         demand_y(x, y, x.get())
-   |         ^^^^^^^^^^^^^^^^^^^^^^^ argument requires that `'1` must outlive `'2`
+   |                        ^^^^^^^ argument requires that `'1` must outlive `'2`
 
 note: No external requirements
   --> $DIR/propagate-fail-to-approximate-longer-wrong-bounds.rs:49:47