From bb30f047f6dbb5c8a73d105b25c2dc23d814b9ba Mon Sep 17 00:00:00 2001 From: Tim Chevalier Date: Fri, 6 Sep 2013 18:59:29 -0700 Subject: [PATCH] workcache: Add the ability to save and load the database... ...as well as the ability to discover inputs and outputs. --- src/libextra/workcache.rs | 158 +++++++++++++++++++++++++++++++++----- 1 file changed, 139 insertions(+), 19 deletions(-) diff --git a/src/libextra/workcache.rs b/src/libextra/workcache.rs index 61af5cd7839..bf897e93881 100644 --- a/src/libextra/workcache.rs +++ b/src/libextra/workcache.rs @@ -12,17 +12,15 @@ use digest::Digest; use json; +use json::ToJson; use sha1::Sha1; use serialize::{Encoder, Encodable, Decoder, Decodable}; use arc::{Arc,RWArc}; use treemap::TreeMap; - use std::cell::Cell; use std::comm::{PortOne, oneshot}; use std::either::{Either, Left, Right}; -use std::io; -use std::run; -use std::task; +use std::{io, os, task}; /** * @@ -107,11 +105,27 @@ impl WorkKey { } } +// FIXME #8883: The key should be a WorkKey and not a ~str. +// This is working around some JSON weirdness. #[deriving(Clone, Eq, Encodable, Decodable)] -struct WorkMap(TreeMap); +struct WorkMap(TreeMap<~str, KindMap>); + +#[deriving(Clone, Eq, Encodable, Decodable)] +struct KindMap(TreeMap<~str, ~str>); impl WorkMap { fn new() -> WorkMap { WorkMap(TreeMap::new()) } + + fn insert_work_key(&mut self, k: WorkKey, val: ~str) { + let WorkKey { kind, name } = k; + match self.find_mut(&name) { + Some(&KindMap(ref mut m)) => { m.insert(kind, val); return; } + None => () + } + let mut new_map = TreeMap::new(); + new_map.insert(kind, val); + self.insert(name, KindMap(new_map)); + } } struct Database { @@ -123,11 +137,15 @@ struct Database { impl Database { pub fn new(p: Path) -> Database { - Database { + let mut rslt = Database { db_filename: p, db_cache: TreeMap::new(), db_dirty: false + }; + if os::path_exists(&rslt.db_filename) { + rslt.load(); } + rslt } pub fn prepare(&self, @@ -154,6 +172,41 @@ impl Database { self.db_cache.insert(k,v); self.db_dirty = true } + + // FIXME #4330: This should have &mut self and should set self.db_dirty to false. + fn save(&self) { + let f = io::file_writer(&self.db_filename, [io::Create, io::Truncate]).unwrap(); + self.db_cache.to_json().to_pretty_writer(f); + } + + fn load(&mut self) { + assert!(!self.db_dirty); + assert!(os::path_exists(&self.db_filename)); + let f = io::file_reader(&self.db_filename); + match f { + Err(e) => fail!("Couldn't load workcache database %s: %s", + self.db_filename.to_str(), e.to_str()), + Ok(r) => + match json::from_reader(r) { + Err(e) => fail!("Couldn't parse workcache database (from file %s): %s", + self.db_filename.to_str(), e.to_str()), + Ok(r) => { + let mut decoder = json::Decoder(r); + self.db_cache = Decodable::decode(&mut decoder); + } + } + } + } +} + +// FIXME #4330: use &mut self here +#[unsafe_destructor] +impl Drop for Database { + fn drop(&self) { + if self.db_dirty { + self.save(); + } + } } struct Logger { @@ -172,12 +225,20 @@ impl Logger { } } +type FreshnessMap = TreeMap<~str,extern fn(&str,&str)->bool>; + #[deriving(Clone)] struct Context { db: RWArc, logger: RWArc, cfg: Arc, - freshness: Arcbool>> + /// Map from kinds (source, exe, url, etc.) to a freshness function. + /// The freshness function takes a name (e.g. file path) and value + /// (e.g. hash of file contents) and determines whether it's up-to-date. + /// For example, in the file case, this would read the file off disk, + /// hash it, and return the result of comparing the given hash and the + /// read hash for equality. + freshness: Arc } struct Prep<'self> { @@ -205,6 +266,7 @@ fn json_encode>(t: &T) -> ~str { // FIXME(#5121) fn json_decode>(s: &str) -> T { + debug!("json decoding: %s", s); do io::with_str_reader(s) |rdr| { let j = json::from_reader(rdr).unwrap(); let mut decoder = json::Decoder(j); @@ -230,11 +292,18 @@ impl Context { pub fn new(db: RWArc, lg: RWArc, cfg: Arc) -> Context { + Context::new_with_freshness(db, lg, cfg, Arc::new(TreeMap::new())) + } + + pub fn new_with_freshness(db: RWArc, + lg: RWArc, + cfg: Arc, + freshness: Arc) -> Context { Context { db: db, logger: lg, cfg: cfg, - freshness: Arc::new(TreeMap::new()) + freshness: freshness } } @@ -249,6 +318,35 @@ impl Context { } +impl Exec { + pub fn discover_input(&mut self, dependency_kind:&str, + // Discovered input + dependency_name: &str, dependency_val: &str) { + debug!("Discovering input %s %s %s", dependency_kind, dependency_name, dependency_val); + self.discovered_inputs.insert_work_key(WorkKey::new(dependency_kind, dependency_name), + dependency_val.to_owned()); + } + pub fn discover_output(&mut self, dependency_kind:&str, + // Discovered output + dependency_name: &str, dependency_val: &str) { + debug!("Discovering output %s %s %s", dependency_kind, dependency_name, dependency_val); + self.discovered_outputs.insert_work_key(WorkKey::new(dependency_kind, dependency_name), + dependency_val.to_owned()); + } + + // returns pairs of (kind, name) + pub fn lookup_discovered_inputs(&self) -> ~[(~str, ~str)] { + let mut rs = ~[]; + for (k, v) in self.discovered_inputs.iter() { + for (k1, _) in v.iter() { + rs.push((k1.clone(), k.clone())); + } + } + rs + } + +} + impl<'self> Prep<'self> { fn new(ctxt: &'self Context, fn_name: &'self str) -> Prep<'self> { Prep { @@ -257,11 +355,22 @@ impl<'self> Prep<'self> { declared_inputs: WorkMap::new() } } + + pub fn lookup_declared_inputs(&self) -> ~[~str] { + let mut rs = ~[]; + for (_, v) in self.declared_inputs.iter() { + for (inp, _) in v.iter() { + rs.push(inp.clone()); + } + } + rs + } } impl<'self> Prep<'self> { - fn declare_input(&mut self, kind:&str, name:&str, val:&str) { - self.declared_inputs.insert(WorkKey::new(kind, name), + pub fn declare_input(&mut self, kind:&str, name:&str, val:&str) { + debug!("Declaring input %s %s %s", kind, name, val); + self.declared_inputs.insert_work_key(WorkKey::new(kind, name), val.to_owned()); } @@ -269,6 +378,7 @@ impl<'self> Prep<'self> { name: &str, val: &str) -> bool { let k = kind.to_owned(); let f = self.ctxt.freshness.get().find(&k); + debug!("freshness for: %s/%s/%s/%s", cat, kind, name, val) let fresh = match f { None => fail!("missing freshness-function for '%s'", kind), Some(f) => (*f)(name, val) @@ -286,27 +396,31 @@ impl<'self> Prep<'self> { } fn all_fresh(&self, cat: &str, map: &WorkMap) -> bool { - for (k, v) in map.iter() { - if ! self.is_fresh(cat, k.kind, k.name, *v) { - return false; + for (k_name, kindmap) in map.iter() { + for (k_kind, v) in kindmap.iter() { + if ! self.is_fresh(cat, *k_kind, *k_name, *v) { + return false; } + } } return true; } - fn exec + Decodable>( - &'self self, blk: ~fn(&Exec) -> T) -> T { + &'self self, blk: ~fn(&mut Exec) -> T) -> T { self.exec_work(blk).unwrap() } fn exec_work + Decodable>( // FIXME(#5121) - &'self self, blk: ~fn(&Exec) -> T) -> Work<'self, T> { + &'self self, blk: ~fn(&mut Exec) -> T) -> Work<'self, T> { let mut bo = Some(blk); + debug!("exec_work: looking up %s and %?", self.fn_name, + self.declared_inputs); let cached = do self.ctxt.db.read |db| { db.prepare(self.fn_name, &self.declared_inputs) }; @@ -316,21 +430,26 @@ impl<'self> Prep<'self> { if self.all_fresh("declared input",&self.declared_inputs) && self.all_fresh("discovered input", disc_in) && self.all_fresh("discovered output", disc_out) => { + debug!("Cache hit!"); + debug!("Trying to decode: %? / %? / %?", + disc_in, disc_out, *res); Left(json_decode(*res)) } _ => { + debug!("Cache miss!"); let (port, chan) = oneshot(); let blk = bo.take_unwrap(); let chan = Cell::new(chan); +// What happens if the task fails? do task::spawn { - let exe = Exec { + let mut exe = Exec { discovered_inputs: WorkMap::new(), discovered_outputs: WorkMap::new(), }; let chan = chan.take(); - let v = blk(&exe); + let v = blk(&mut exe); chan.send((exe, v)); } Right(port) @@ -371,9 +490,10 @@ impl<'self, T:Send + } -//#[test] +#[test] fn test() { use std::io::WriterUtil; + use std::run; let pth = Path("foo.c"); {