OpenE2K/rust
2
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

workcache: Add the ability to save and load the database...

Browse Source 
...as well as the ability to discover inputs and outputs.
This commit is contained in:
Tim Chevalier 2013-09-06 18:59:29 -07:00
parent dd5c7379e9
commit bb30f047f6
1 changed files with 139 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

158
src/libextra/workcache.rs
View File

@ -12,17 +12,15 @@
use digest::Digest;
use json;
use json::ToJson;
use sha1::Sha1;
use serialize::{Encoder, Encodable, Decoder, Decodable};
use arc::{Arc,RWArc};
use treemap::TreeMap;
use std::cell::Cell;
use std::comm::{PortOne, oneshot};
use std::either::{Either, Left, Right};
use std::io;
use std::run;
use std::task;
use std::{io, os, task};
/**
*
@ -107,11 +105,27 @@ impl WorkKey {
}
}
// FIXME #8883: The key should be a WorkKey and not a ~str.
// This is working around some JSON weirdness.
#[deriving(Clone, Eq, Encodable, Decodable)]
struct WorkMap(TreeMap<WorkKey, ~str>);
struct WorkMap(TreeMap<~str, KindMap>);
#[deriving(Clone, Eq, Encodable, Decodable)]
struct KindMap(TreeMap<~str, ~str>);
impl WorkMap {
fn new() -> WorkMap { WorkMap(TreeMap::new()) }
fn insert_work_key(&mut self, k: WorkKey, val: ~str) {
let WorkKey { kind, name } = k;
match self.find_mut(&name) {
Some(&KindMap(ref mut m)) => { m.insert(kind, val); return; }
None => ()
}
let mut new_map = TreeMap::new();
new_map.insert(kind, val);
self.insert(name, KindMap(new_map));
}
}
struct Database {
@ -123,11 +137,15 @@ struct Database {
impl Database {
pub fn new(p: Path) -> Database {
Database {
let mut rslt = Database {
db_filename: p,
db_cache: TreeMap::new(),
db_dirty: false
};
if os::path_exists(&rslt.db_filename) {
rslt.load();
}
rslt
}
pub fn prepare(&self,
@ -154,6 +172,41 @@ impl Database {
self.db_cache.insert(k,v);
self.db_dirty = true
}
// FIXME #4330: This should have &mut self and should set self.db_dirty to false.
fn save(&self) {
let f = io::file_writer(&self.db_filename, [io::Create, io::Truncate]).unwrap();
self.db_cache.to_json().to_pretty_writer(f);
}
fn load(&mut self) {
assert!(!self.db_dirty);
assert!(os::path_exists(&self.db_filename));
let f = io::file_reader(&self.db_filename);
match f {
Err(e) => fail!("Couldn't load workcache database %s: %s",
self.db_filename.to_str(), e.to_str()),
Ok(r) =>
match json::from_reader(r) {
Err(e) => fail!("Couldn't parse workcache database (from file %s): %s",
self.db_filename.to_str(), e.to_str()),
Ok(r) => {
let mut decoder = json::Decoder(r);
self.db_cache = Decodable::decode(&mut decoder);
}
}
}
}
}
// FIXME #4330: use &mut self here
#[unsafe_destructor]
impl Drop for Database {
fn drop(&self) {
if self.db_dirty {
self.save();
}
}
}
struct Logger {
@ -172,12 +225,20 @@ impl Logger {
}
}
type FreshnessMap = TreeMap<~str,extern fn(&str,&str)->bool>;
#[deriving(Clone)]
struct Context {
db: RWArc<Database>,
logger: RWArc<Logger>,
cfg: Arc<json::Object>,
freshness: Arc<TreeMap<~str,extern fn(&str,&str)->bool>>
/// Map from kinds (source, exe, url, etc.) to a freshness function.
/// The freshness function takes a name (e.g. file path) and value
/// (e.g. hash of file contents) and determines whether it's up-to-date.
/// For example, in the file case, this would read the file off disk,
/// hash it, and return the result of comparing the given hash and the
/// read hash for equality.
freshness: Arc<FreshnessMap>
}
struct Prep<'self> {
@ -205,6 +266,7 @@ fn json_encode<T:Encodable<json::Encoder>>(t: &T) -> ~str {
// FIXME(#5121)
fn json_decode<T:Decodable<json::Decoder>>(s: &str) -> T {
debug!("json decoding: %s", s);
do io::with_str_reader(s) |rdr| {
let j = json::from_reader(rdr).unwrap();
let mut decoder = json::Decoder(j);
@ -230,11 +292,18 @@ impl Context {
pub fn new(db: RWArc<Database>,
lg: RWArc<Logger>,
cfg: Arc<json::Object>) -> Context {
Context::new_with_freshness(db, lg, cfg, Arc::new(TreeMap::new()))
}
pub fn new_with_freshness(db: RWArc<Database>,
lg: RWArc<Logger>,
cfg: Arc<json::Object>,
freshness: Arc<FreshnessMap>) -> Context {
Context {
db: db,
logger: lg,
cfg: cfg,
freshness: Arc::new(TreeMap::new())
freshness: freshness
}
}
@ -249,6 +318,35 @@ impl Context {
}
impl Exec {
pub fn discover_input(&mut self, dependency_kind:&str,
// Discovered input
dependency_name: &str, dependency_val: &str) {
debug!("Discovering input %s %s %s", dependency_kind, dependency_name, dependency_val);
self.discovered_inputs.insert_work_key(WorkKey::new(dependency_kind, dependency_name),
dependency_val.to_owned());
}
pub fn discover_output(&mut self, dependency_kind:&str,
// Discovered output
dependency_name: &str, dependency_val: &str) {
debug!("Discovering output %s %s %s", dependency_kind, dependency_name, dependency_val);
self.discovered_outputs.insert_work_key(WorkKey::new(dependency_kind, dependency_name),
dependency_val.to_owned());
}
// returns pairs of (kind, name)
pub fn lookup_discovered_inputs(&self) -> ~[(~str, ~str)] {
let mut rs = ~[];
for (k, v) in self.discovered_inputs.iter() {
for (k1, _) in v.iter() {
rs.push((k1.clone(), k.clone()));
}
}
rs
}
}
impl<'self> Prep<'self> {
fn new(ctxt: &'self Context, fn_name: &'self str) -> Prep<'self> {
Prep {
@ -257,11 +355,22 @@ impl<'self> Prep<'self> {
declared_inputs: WorkMap::new()
}
}
pub fn lookup_declared_inputs(&self) -> ~[~str] {
let mut rs = ~[];
for (_, v) in self.declared_inputs.iter() {
for (inp, _) in v.iter() {
rs.push(inp.clone());
}
}
rs
}
}
impl<'self> Prep<'self> {
fn declare_input(&mut self, kind:&str, name:&str, val:&str) {
self.declared_inputs.insert(WorkKey::new(kind, name),
pub fn declare_input(&mut self, kind:&str, name:&str, val:&str) {
debug!("Declaring input %s %s %s", kind, name, val);
self.declared_inputs.insert_work_key(WorkKey::new(kind, name),
val.to_owned());
}
@ -269,6 +378,7 @@ impl<'self> Prep<'self> {
name: &str, val: &str) -> bool {
let k = kind.to_owned();
let f = self.ctxt.freshness.get().find(&k);
debug!("freshness for: %s/%s/%s/%s", cat, kind, name, val)
let fresh = match f {
None => fail!("missing freshness-function for '%s'", kind),
Some(f) => (*f)(name, val)
@ -286,27 +396,31 @@ impl<'self> Prep<'self> {
}
fn all_fresh(&self, cat: &str, map: &WorkMap) -> bool {
for (k, v) in map.iter() {
if ! self.is_fresh(cat, k.kind, k.name, *v) {
return false;
for (k_name, kindmap) in map.iter() {
for (k_kind, v) in kindmap.iter() {
if ! self.is_fresh(cat, *k_kind, *k_name, *v) {
return false;
}
}
}
return true;
}
fn exec<T:Send +
pub fn exec<T:Send +
Encodable<json::Encoder> +
Decodable<json::Decoder>>(
&'self self, blk: ~fn(&Exec) -> T) -> T {
&'self self, blk: ~fn(&mut Exec) -> T) -> T {
self.exec_work(blk).unwrap()
}
fn exec_work<T:Send +
Encodable<json::Encoder> +
Decodable<json::Decoder>>( // FIXME(#5121)
&'self self, blk: ~fn(&Exec) -> T) -> Work<'self, T> {
&'self self, blk: ~fn(&mut Exec) -> T) -> Work<'self, T> {
let mut bo = Some(blk);
debug!("exec_work: looking up %s and %?", self.fn_name,
self.declared_inputs);
let cached = do self.ctxt.db.read |db| {
db.prepare(self.fn_name, &self.declared_inputs)
};
@ -316,21 +430,26 @@ impl<'self> Prep<'self> {
if self.all_fresh("declared input",&self.declared_inputs) &&
self.all_fresh("discovered input", disc_in) &&
self.all_fresh("discovered output", disc_out) => {
debug!("Cache hit!");
debug!("Trying to decode: %? / %? / %?",
disc_in, disc_out, *res);
Left(json_decode(*res))
}
_ => {
debug!("Cache miss!");
let (port, chan) = oneshot();
let blk = bo.take_unwrap();
let chan = Cell::new(chan);
// What happens if the task fails?
do task::spawn {
let exe = Exec {
let mut exe = Exec {
discovered_inputs: WorkMap::new(),
discovered_outputs: WorkMap::new(),
};
let chan = chan.take();
let v = blk(&exe);
let v = blk(&mut exe);
chan.send((exe, v));
}
Right(port)
@ -371,9 +490,10 @@ impl<'self, T:Send +
}
//#[test]
#[test]
fn test() {
use std::io::WriterUtil;
use std::run;
let pth = Path("foo.c");
{