From 1996a11bb29b61802d18830ba1eabdf4f99582ae Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Fri, 4 Oct 2013 15:04:11 -0700 Subject: [PATCH 1/2] Cache more results of const_eval According to http://huonw.github.io/isrustfastyet/mem/#012f909, the "const marking" pass generates about 400MB of extra memory during compilation. It appears that this is due to two different factors: 1. There is a `ccache` map in the ty::ctxt which is only ever used in this pass, so this commit moves the map out of the ty::ctxt struct and into just this pass's visitor. This turned out to not benefit that much in memory (as indicated by http://i.imgur.com/Eo4iOzK.png), but it's helpful to do nonetheless. 2. During const_eval, there are a lot of lookups into decoding inlined items from external crates. There is no caching involved here, so the same static or variant could be re-translated many times. After adding separate caches for variants and statics, the memory peak of compiling rustc decreased by 200MB (as evident by http://i.imgur.com/ULAUMtq.png) The culmination of this is basically a slight reorganization of a caching map for the const_eval pass along with a 200MB decrease in peak memory usage when compiling librustc. --- src/librustc/middle/const_eval.rs | 231 +++++++++++++++--------------- src/librustc/middle/ty.rs | 12 +- 2 files changed, 126 insertions(+), 117 deletions(-) diff --git a/src/librustc/middle/const_eval.rs b/src/librustc/middle/const_eval.rs index d13493c0a02..cc818c9c001 100644 --- a/src/librustc/middle/const_eval.rs +++ b/src/librustc/middle/const_eval.rs @@ -60,6 +60,8 @@ pub enum constness { non_const } +type constness_cache = HashMap; + pub fn join(a: constness, b: constness) -> constness { match (a, b) { (integral_const, integral_const) => integral_const, @@ -74,102 +76,12 @@ pub fn join_all>(mut cs: It) -> constness { cs.fold(integral_const, |a, b| join(a, b)) } -pub fn classify(e: &Expr, - tcx: ty::ctxt) - -> constness { - let did = ast_util::local_def(e.id); - match tcx.ccache.find(&did) { - Some(&x) => x, - None => { - let cn = - match e.node { - ast::ExprLit(lit) => { - match lit.node { - ast::lit_str(*) | - ast::lit_float(*) => general_const, - _ => integral_const - } - } - - ast::ExprUnary(_, _, inner) | - ast::ExprParen(inner) => { - classify(inner, tcx) - } - - ast::ExprBinary(_, _, a, b) => { - join(classify(a, tcx), - classify(b, tcx)) - } - - ast::ExprTup(ref es) | - ast::ExprVec(ref es, ast::MutImmutable) => { - join_all(es.iter().map(|e| classify(*e, tcx))) - } - - ast::ExprVstore(e, vstore) => { - match vstore { - ast::ExprVstoreSlice => classify(e, tcx), - ast::ExprVstoreUniq | - ast::ExprVstoreBox | - ast::ExprVstoreMutBox | - ast::ExprVstoreMutSlice => non_const - } - } - - ast::ExprStruct(_, ref fs, None) => { - let cs = do fs.iter().map |f| { - classify(f.expr, tcx) - }; - join_all(cs) - } - - ast::ExprCast(base, _) => { - let ty = ty::expr_ty(tcx, e); - let base = classify(base, tcx); - if ty::type_is_integral(ty) { - join(integral_const, base) - } else if ty::type_is_fp(ty) { - join(general_const, base) - } else { - non_const - } - } - - ast::ExprField(base, _, _) => { - classify(base, tcx) - } - - ast::ExprIndex(_, base, idx) => { - join(classify(base, tcx), - classify(idx, tcx)) - } - - ast::ExprAddrOf(ast::MutImmutable, base) => { - classify(base, tcx) - } - - // FIXME: (#3728) we can probably do something CCI-ish - // surrounding nonlocal constants. But we don't yet. - ast::ExprPath(_) => { - lookup_constness(tcx, e) - } - - ast::ExprRepeat(*) => general_const, - - _ => non_const - }; - tcx.ccache.insert(did, cn); - cn - } - } -} - pub fn lookup_const(tcx: ty::ctxt, e: &Expr) -> Option<@Expr> { match tcx.def_map.find(&e.id) { - Some(&ast::DefStatic(def_id, false)) => lookup_const_by_id(tcx, def_id), - Some(&ast::DefVariant(enum_def, variant_def, _)) => lookup_variant_by_id(tcx, - enum_def, - variant_def), + Some(&ast::DefStatic(def_id, false)) => + lookup_const_by_id(tcx, def_id), + Some(&ast::DefVariant(enum_def, variant_def, _)) => + lookup_variant_by_id(tcx, enum_def, variant_def), _ => None } } @@ -199,6 +111,10 @@ pub fn lookup_variant_by_id(tcx: ty::ctxt, Some(_) => None } } else { + match tcx.extern_const_variants.find(&variant_def) { + Some(&e) => return e, + None => {} + } let maps = astencode::Maps { root_map: @mut HashMap::new(), method_map: @mut HashMap::new(), @@ -206,7 +122,7 @@ pub fn lookup_variant_by_id(tcx: ty::ctxt, write_guard_map: @mut HashSet::new(), capture_map: @mut HashMap::new() }; - match csearch::maybe_get_item_ast(tcx, enum_def, + let e = match csearch::maybe_get_item_ast(tcx, enum_def, |a, b, c, d| astencode::decode_inlined_item(a, b, maps, @@ -219,7 +135,9 @@ pub fn lookup_variant_by_id(tcx: ty::ctxt, _ => None }, _ => None - } + }; + tcx.extern_const_variants.insert(variant_def, e); + return e; } } @@ -236,6 +154,10 @@ pub fn lookup_const_by_id(tcx: ty::ctxt, Some(_) => None } } else { + match tcx.extern_const_statics.find(&def_id) { + Some(&e) => return e, + None => {} + } let maps = astencode::Maps { root_map: @mut HashMap::new(), method_map: @mut HashMap::new(), @@ -243,42 +165,125 @@ pub fn lookup_const_by_id(tcx: ty::ctxt, write_guard_map: @mut HashSet::new(), capture_map: @mut HashMap::new() }; - match csearch::maybe_get_item_ast(tcx, def_id, + let e = match csearch::maybe_get_item_ast(tcx, def_id, |a, b, c, d| astencode::decode_inlined_item(a, b, maps, c, d)) { csearch::found(ast::ii_item(item)) => match item.node { item_static(_, ast::MutImmutable, const_expr) => Some(const_expr), _ => None }, _ => None - } + }; + tcx.extern_const_statics.insert(def_id, e); + return e; } } -pub fn lookup_constness(tcx: ty::ctxt, e: &Expr) -> constness { - match lookup_const(tcx, e) { - Some(rhs) => { - let ty = ty::expr_ty(tcx, rhs); - if ty::type_is_integral(ty) { - integral_const - } else { - general_const +struct ConstEvalVisitor { + tcx: ty::ctxt, + ccache: constness_cache, +} + +impl ConstEvalVisitor { + fn classify(&mut self, e: &Expr) -> constness { + let did = ast_util::local_def(e.id); + match self.ccache.find(&did) { + Some(&x) => return x, + None => {} + } + let cn = match e.node { + ast::ExprLit(lit) => { + match lit.node { + ast::lit_str(*) | ast::lit_float(*) => general_const, + _ => integral_const + } } - } - None => non_const - } -} -struct ConstEvalVisitor { tcx: ty::ctxt } + ast::ExprUnary(_, _, inner) | ast::ExprParen(inner) => + self.classify(inner), + + ast::ExprBinary(_, _, a, b) => + join(self.classify(a), self.classify(b)), + + ast::ExprTup(ref es) | + ast::ExprVec(ref es, ast::MutImmutable) => + join_all(es.iter().map(|e| self.classify(*e))), + + ast::ExprVstore(e, vstore) => { + match vstore { + ast::ExprVstoreSlice => self.classify(e), + ast::ExprVstoreUniq | + ast::ExprVstoreBox | + ast::ExprVstoreMutBox | + ast::ExprVstoreMutSlice => non_const + } + } + + ast::ExprStruct(_, ref fs, None) => { + let cs = do fs.iter().map |f| { + self.classify(f.expr) + }; + join_all(cs) + } + + ast::ExprCast(base, _) => { + let ty = ty::expr_ty(self.tcx, e); + let base = self.classify(base); + if ty::type_is_integral(ty) { + join(integral_const, base) + } else if ty::type_is_fp(ty) { + join(general_const, base) + } else { + non_const + } + } + + ast::ExprField(base, _, _) => self.classify(base), + + ast::ExprIndex(_, base, idx) => + join(self.classify(base), self.classify(idx)), + + ast::ExprAddrOf(ast::MutImmutable, base) => self.classify(base), + + // FIXME: (#3728) we can probably do something CCI-ish + // surrounding nonlocal constants. But we don't yet. + ast::ExprPath(_) => self.lookup_constness(e), + + ast::ExprRepeat(*) => general_const, + + _ => non_const + }; + self.ccache.insert(did, cn); + cn + } + + fn lookup_constness(&self, e: &Expr) -> constness { + match lookup_const(self.tcx, e) { + Some(rhs) => { + let ty = ty::expr_ty(self.tcx, rhs); + if ty::type_is_integral(ty) { + integral_const + } else { + general_const + } + } + None => non_const + } + } + +} impl Visitor<()> for ConstEvalVisitor { fn visit_expr_post(&mut self, e:@Expr, _:()) { - classify(e, self.tcx); + self.classify(e); } } pub fn process_crate(crate: &ast::Crate, tcx: ty::ctxt) { - let mut v = ConstEvalVisitor { tcx: tcx }; + let mut v = ConstEvalVisitor { + tcx: tcx, + ccache: HashMap::new(), + }; visit::walk_crate(&mut v, crate, ()); tcx.sess.abort_if_errors(); } diff --git a/src/librustc/middle/ty.rs b/src/librustc/middle/ty.rs index bd099e81a17..d21852751b4 100644 --- a/src/librustc/middle/ty.rs +++ b/src/librustc/middle/ty.rs @@ -290,7 +290,6 @@ struct ctxt_ { freevars: freevars::freevar_map, tcache: type_cache, rcache: creader_cache, - ccache: constness_cache, short_names_cache: @mut HashMap, needs_unwind_cleanup_cache: @mut HashMap, tc_cache: @mut HashMap, @@ -346,6 +345,11 @@ struct ctxt_ { // The set of external traits whose implementations have been read. This // is used for lazy resolution of traits. populated_external_traits: @mut HashSet, + + // These two caches are used by const_eval when decoding external statics + // and variants that are found. + extern_const_statics: @mut HashMap>, + extern_const_variants: @mut HashMap>, } pub enum tbox_flag { @@ -897,8 +901,6 @@ pub struct ty_param_substs_and_ty { type type_cache = @mut HashMap; -type constness_cache = @mut HashMap; - pub type node_type_table = @mut HashMap; fn mk_rcache() -> creader_cache { @@ -935,7 +937,6 @@ pub fn mk_ctxt(s: session::Session, freevars: freevars, tcache: @mut HashMap::new(), rcache: mk_rcache(), - ccache: @mut HashMap::new(), short_names_cache: new_ty_hash(), needs_unwind_cleanup_cache: new_ty_hash(), tc_cache: @mut HashMap::new(), @@ -961,6 +962,9 @@ pub fn mk_ctxt(s: session::Session, impl_vtables: @mut HashMap::new(), populated_external_types: @mut HashSet::new(), populated_external_traits: @mut HashSet::new(), + + extern_const_statics: @mut HashMap::new(), + extern_const_variants: @mut HashMap::new(), } } From 19e9766c29e3e958409d7232eb0c23b3734988aa Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Fri, 4 Oct 2013 21:16:54 -0700 Subject: [PATCH 2/2] Don't fail when unpacking the windows snapshot Newly having a third-party directory was throwing off the unpack script --- src/etc/get-snapshot.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/etc/get-snapshot.py b/src/etc/get-snapshot.py index af1a6d6da7b..2e547dbbcd1 100755 --- a/src/etc/get-snapshot.py +++ b/src/etc/get-snapshot.py @@ -26,6 +26,8 @@ def unpack_snapshot(triple, dl_path): print("extracting " + p) tar.extract(p, download_unpack_base) tp = os.path.join(download_unpack_base, p) + if os.path.isdir(tp) and os.path.exists(fp): + continue shutil.move(tp, fp) tar.close() shutil.rmtree(download_unpack_base)