diff --git a/src/libextra/workcache.rs b/src/libextra/workcache.rs index bf897e93881..d5d37a30f6b 100644 --- a/src/libextra/workcache.rs +++ b/src/libextra/workcache.rs @@ -493,7 +493,7 @@ impl<'self, T:Send + #[test] fn test() { use std::io::WriterUtil; - use std::run; + use std::{os, run}; let pth = Path("foo.c"); { @@ -501,7 +501,12 @@ fn test() { r.unwrap().write_str("int main() { return 0; }"); } - let cx = Context::new(RWArc::new(Database::new(Path("db.json"))), + let db_path = os::self_exe_path().expect("workcache::test failed").pop().push("db.json"); + if os::path_exists(&db_path) { + os::remove_file(&db_path); + } + + let cx = Context::new(RWArc::new(Database::new(db_path)), RWArc::new(Logger::new()), Arc::new(TreeMap::new())); diff --git a/src/librustpkg/api.rs b/src/librustpkg/api.rs index dfe80674b7f..42e5668489a 100644 --- a/src/librustpkg/api.rs +++ b/src/librustpkg/api.rs @@ -13,49 +13,101 @@ use crate::*; use package_id::*; use package_source::*; use version::Version; +use workcache_support::*; +use extra::arc::{Arc,RWArc}; +use extra::workcache; +use extra::workcache::*; use std::os; -use std::hashmap::*; +use extra::treemap::TreeMap; /// Convenience functions intended for calling from pkg.rs +/// p is where to put the cache file for dependencies +pub fn default_ctxt(p: Path) -> BuildCtx { + new_default_ctx(new_workcache_cx(&p), p) +} -fn default_ctxt(p: @Path) -> Ctx { - Ctx { - use_rust_path_hack: false, - sysroot_opt: Some(p), - json: false, - dep_cache: @mut HashMap::new() +pub fn new_default_ctx(c: Context, p: Path) -> BuildCtx { + BuildCtx { + cx: Ctx { use_rust_path_hack: false, + sysroot_opt: p }, + workcache_cx: c } } -pub fn build_lib(sysroot: @Path, root: Path, name: ~str, version: Version, +fn file_is_fresh(path: &str, in_hash: &str) -> bool { + in_hash == digest_file_with_date(&Path(path)) +} + +fn binary_is_fresh(path: &str, in_hash: &str) -> bool { + in_hash == digest_only_date(&Path(path)) +} + + +pub fn new_workcache_cx(p: &Path) -> Context { + let db_file = p.push("rustpkg_db.json"); // ??? probably wrong + debug!("Workcache database file: %s", db_file.to_str()); + let db = RWArc::new(Database::new(db_file)); + let lg = RWArc::new(Logger::new()); + let cfg = Arc::new(TreeMap::new()); + let mut rslt: FreshnessMap = TreeMap::new(); + // Set up freshness functions for every type of dependency rustpkg + // knows about + rslt.insert(~"file", file_is_fresh); + rslt.insert(~"binary", binary_is_fresh); + workcache::Context::new_with_freshness(db, lg, cfg, Arc::new(rslt)) +} + +pub fn build_lib(sysroot: Path, root: Path, name: ~str, version: Version, lib: Path) { - - let pkg_src = PkgSrc { - root: root, - id: PkgId{ version: version, ..PkgId::new(name)}, - libs: ~[mk_crate(lib)], - mains: ~[], - tests: ~[], - benchs: ~[] + let cx = default_ctxt(sysroot); + let subroot = root.clone(); + let subversion = version.clone(); + let sublib = lib.clone(); + do cx.workcache_cx.with_prep(name) |prep| { + let pkg_src = PkgSrc { + workspace: subroot.clone(), + start_dir: subroot.push("src").push(name), + id: PkgId{ version: subversion.clone(), ..PkgId::new(name)}, + libs: ~[mk_crate(sublib.clone())], + mains: ~[], + tests: ~[], + benchs: ~[] + }; + pkg_src.declare_inputs(prep); + let subcx = cx.clone(); + let subsrc = pkg_src.clone(); + do prep.exec |exec| { + subsrc.clone().build(exec, &subcx.clone(), ~[]); + } }; - pkg_src.build(&default_ctxt(sysroot), ~[]); } -pub fn build_exe(sysroot: @Path, root: Path, name: ~str, version: Version, main: Path) { - let pkg_src = PkgSrc { - root: root, - id: PkgId{ version: version, ..PkgId::new(name)}, - libs: ~[], - mains: ~[mk_crate(main)], - tests: ~[], - benchs: ~[] - }; - pkg_src.build(&default_ctxt(sysroot), ~[]); - +pub fn build_exe(sysroot: Path, root: Path, name: ~str, version: Version, + main: Path) { + let cx = default_ctxt(sysroot); + let subroot = root.clone(); + let submain = main.clone(); + do cx.workcache_cx.with_prep(name) |prep| { + let pkg_src = PkgSrc { + workspace: subroot.clone(), + start_dir: subroot.push("src").push(name), + id: PkgId{ version: version.clone(), ..PkgId::new(name)}, + libs: ~[], + mains: ~[mk_crate(submain.clone())], + tests: ~[], + benchs: ~[] + }; + pkg_src.declare_inputs(prep); + let subsrc = pkg_src.clone(); + let subcx = cx.clone(); + do prep.exec |exec| { + subsrc.clone().build(exec, &subcx.clone(), ~[]); + } + } } -pub fn install_lib(sysroot: @Path, +pub fn install_lib(sysroot: Path, workspace: Path, name: ~str, lib_path: Path, @@ -65,23 +117,33 @@ pub fn install_lib(sysroot: @Path, debug!("workspace = %s", workspace.to_str()); // make a PkgSrc let pkg_id = PkgId{ version: version, ..PkgId::new(name)}; - let pkg_src = PkgSrc { - root: workspace.clone(), - id: pkg_id.clone(), - libs: ~[mk_crate(lib_path)], - mains: ~[], - tests: ~[], - benchs: ~[] - }; let cx = default_ctxt(sysroot); - pkg_src.build(&cx, ~[]); + let subpath = lib_path.clone(); + do cx.workcache_cx.with_prep(pkg_id.to_str()) |prep| { + let pkg_src = PkgSrc { + workspace: workspace.clone(), + start_dir: subpath.push("src").push(name), + id: pkg_id.clone(), + libs: ~[mk_crate(subpath.clone())], + mains: ~[], + tests: ~[], + benchs: ~[] + }; + pkg_src.declare_inputs(prep); + let subcx = cx.clone(); + let subpkg_src = pkg_src.clone(); + do prep.exec |exec| { + subpkg_src.clone().build(exec, &subcx.clone(), ~[]); + } + } cx.install_no_build(&workspace, &pkg_id); } -pub fn install_exe(sysroot: @Path, workspace: Path, name: ~str, version: Version) { - default_ctxt(sysroot).install(&workspace, &PkgId{ version: version, - ..PkgId::new(name)}); - +pub fn install_exe(sysroot: Path, workspace: Path, name: ~str, version: Version) { + let cx = default_ctxt(sysroot); + debug!("install_exe calling with_prep"); + let pkgid = PkgId{ version: version, ..PkgId::new(name)}; + cx.install(PkgSrc::new(workspace, false, pkgid)); } fn mk_crate(p: Path) -> Crate { diff --git a/src/librustpkg/conditions.rs b/src/librustpkg/conditions.rs index 4cb103deba7..2e049d252bc 100644 --- a/src/librustpkg/conditions.rs +++ b/src/librustpkg/conditions.rs @@ -12,11 +12,17 @@ pub use std::path::Path; pub use package_id::PkgId; +pub use std::libc; +pub use std::libc::stat; condition! { pub bad_path: (Path, ~str) -> Path; } +condition! { + pub bad_stat: (Path, ~str) -> stat; +} + condition! { pub nonexistent_package: (PkgId, ~str) -> Path; } diff --git a/src/librustpkg/context.rs b/src/librustpkg/context.rs index 4087fdd7ca5..70f1e39efec 100644 --- a/src/librustpkg/context.rs +++ b/src/librustpkg/context.rs @@ -10,56 +10,66 @@ // Context data structure used by rustpkg - -use std::hashmap::HashMap; use std::os; +use extra::workcache; +#[deriving(Clone)] pub struct Ctx { // If use_rust_path_hack is true, rustpkg searches for sources // in *package* directories that are in the RUST_PATH (for example, // FOO/src/bar-0.1 instead of FOO). The flag doesn't affect where // rustpkg stores build artifacts. use_rust_path_hack: bool, - // Sysroot -- if this is None, uses rustc filesearch's - // idea of the default - sysroot_opt: Option<@Path>, - // I'm not sure what this is for - json: bool, - // Cache of hashes of things already installed - // though I'm not sure why the value is a bool - dep_cache: @mut HashMap<~str, bool>, + // The root directory containing the Rust standard libraries + sysroot_opt: Path +} + +#[deriving(Clone)] +pub struct BuildCtx { + // Context for workcache + workcache_cx: workcache::Context, + // Everything else + cx: Ctx +} + +impl BuildCtx { + pub fn sysroot_opt(&self) -> Path { + self.cx.sysroot_opt.clone() + } + + pub fn sysroot_to_use(&self) -> Path { + self.cx.sysroot_to_use() + } +} + +impl Ctx { + pub fn sysroot_opt(&self) -> Path { + self.sysroot_opt.clone() + } } impl Ctx { /// Debugging pub fn sysroot_opt_str(&self) -> ~str { - match self.sysroot_opt { - None => ~"[none]", - Some(p) => p.to_str() - } + self.sysroot_opt.to_str() } // Hack so that rustpkg can run either out of a rustc target dir, // or the host dir - pub fn sysroot_to_use(&self) -> Option<@Path> { - if !in_target(self.sysroot_opt) { - self.sysroot_opt + pub fn sysroot_to_use(&self) -> Path { + if !in_target(&self.sysroot_opt) { + self.sysroot_opt.clone() } else { - self.sysroot_opt.map(|p| { @p.pop().pop().pop() }) + self.sysroot_opt.pop().pop().pop() } - } + } } /// We assume that if ../../rustc exists, then we're running /// rustpkg from a Rust target directory. This is part of a /// kludgy hack used to adjust the sysroot. -pub fn in_target(sysroot_opt: Option<@Path>) -> bool { - match sysroot_opt { - None => false, - Some(p) => { - debug!("Checking whether %s is in target", p.to_str()); - os::path_is_dir(&p.pop().pop().push("rustc")) - } - } +pub fn in_target(sysroot_opt: &Path) -> bool { + debug!("Checking whether %s is in target", sysroot_opt.to_str()); + os::path_is_dir(&sysroot_opt.pop().pop().push("rustc")) } diff --git a/src/librustpkg/package_id.rs b/src/librustpkg/package_id.rs index ce1da58a2cb..f0f3673f1d0 100644 --- a/src/librustpkg/package_id.rs +++ b/src/librustpkg/package_id.rs @@ -51,12 +51,10 @@ impl PkgId { // Did the user request a specific version? let s = match split_version(s) { Some((path, v)) => { - debug!("s = %s, path = %s, v = %s", s, path, v.to_str()); given_version = Some(v); path } None => { - debug!("%s has no explicit version", s); s } }; @@ -81,7 +79,6 @@ impl PkgId { } }; - debug!("path = %s", path.to_str()); PkgId { path: path.clone(), short_name: short_name.to_owned(), diff --git a/src/librustpkg/package_source.rs b/src/librustpkg/package_source.rs index c20091158c4..45932283081 100644 --- a/src/librustpkg/package_source.rs +++ b/src/librustpkg/package_source.rs @@ -18,14 +18,19 @@ use context::*; use crate::Crate; use messages::*; use source_control::{git_clone, git_clone_general}; -use path_util::{pkgid_src_in_workspace, find_dir_using_rust_path_hack, default_workspace}; +use path_util::{find_dir_using_rust_path_hack, default_workspace}; use util::compile_crate; use workspace::is_workspace; +use workcache_support; +use extra::workcache; // An enumeration of the unpacked source of a package workspace. // This contains a list of files found in the source workspace. +#[deriving(Clone)] pub struct PkgSrc { - root: Path, // root of where the package source code lives + workspace: Path, // root of where the package source code lives + start_dir: Path, // dir to start looking in for packages -- normally + // this is workspace/src/id but it may be just workspace id: PkgId, libs: ~[Crate], mains: ~[Crate], @@ -33,17 +38,86 @@ pub struct PkgSrc { benchs: ~[Crate], } +impl ToStr for PkgSrc { + fn to_str(&self) -> ~str { + fmt!("Package ID %s in start dir %s [workspace = %s]", + self.id.to_str(), + self.start_dir.to_str(), self.workspace.to_str()) + } +} condition! { // #6009: should this be pub or not, when #8215 is fixed? - build_err: (~str) -> (); + build_err: (~str) -> ~str; } impl PkgSrc { - pub fn new(src_dir: &Path, id: &PkgId) -> PkgSrc { + pub fn new(workspace: Path, use_rust_path_hack: bool, id: PkgId) -> PkgSrc { + use conditions::nonexistent_package::cond; + + debug!("Checking package source for package ID %s, \ + workspace = %s", id.to_str(), workspace.to_str()); + + let mut to_try = ~[]; + if use_rust_path_hack { + to_try.push(workspace.clone()); + } + else { + let result = workspace.push("src").push_rel(&id.path.pop()).push(fmt!("%s-%s", + id.short_name, id.version.to_str())); + to_try.push(result); + to_try.push(workspace.push("src").push_rel(&id.path)); + } + + debug!("Checking dirs: %?", to_try.map(|s| s.to_str()).connect(":")); + + let path = to_try.iter().find(|&d| os::path_exists(d)); + + let dir: Path = match path { + Some(d) => (*d).clone(), + None => { + let mut ok_d = None; + for w in to_try.iter() { + debug!("Calling fetch_git on %s", w.to_str()); + let gf = PkgSrc::fetch_git(w, &id); + for p in gf.iter() { + ok_d = Some(p.clone()); + break; + } + if ok_d.is_some() { break; } + } + match ok_d { + Some(d) => d, + None => { + if use_rust_path_hack { + match find_dir_using_rust_path_hack(&id) { + Some(d) => d, + None => cond.raise((id.clone(), + ~"supplied path for package dir does not \ + exist, and couldn't interpret it as a URL fragment")) + } + } + else { + cond.raise((id.clone(), + ~"supplied path for package dir does not \ + exist, and couldn't interpret it as a URL fragment")) + + } + } + } + } + }; + debug!("For package id %s, returning %s", id.to_str(), dir.to_str()); + + if !os::path_is_dir(&dir) { + cond.raise((id.clone(), ~"supplied path for package dir is a \ + non-directory")); + } + PkgSrc { - root: (*src_dir).clone(), - id: (*id).clone(), + workspace: workspace, + start_dir: dir, + id: id, libs: ~[], mains: ~[], tests: ~[], @@ -51,47 +125,12 @@ impl PkgSrc { } } - - fn check_dir(&self, cx: &Ctx) -> Path { - use conditions::nonexistent_package::cond; - - debug!("Pushing onto root: %s | %s", self.id.path.to_str(), self.root.to_str()); - - let dirs = pkgid_src_in_workspace(&self.id, &self.root); - debug!("Checking dirs: %?", dirs.map(|s| s.to_str()).connect(":")); - let path = dirs.iter().find(|&d| os::path_exists(d)); - - let dir = match path { - Some(d) => (*d).clone(), - None => { - match self.fetch_git() { - Some(d) => d, - None => { - match find_dir_using_rust_path_hack(cx, &self.id) { - Some(d) => d, - None => cond.raise((self.id.clone(), - ~"supplied path for package dir does not \ - exist, and couldn't interpret it as a URL fragment")) - } - } - } - } - }; - debug!("For package id %s, returning %s", self.id.to_str(), dir.to_str()); - if !os::path_is_dir(&dir) { - cond.raise((self.id.clone(), ~"supplied path for package dir is a \ - non-directory")); - } - - dir - } - /// Try interpreting self's package id as a git repository, and try /// fetching it and caching it in a local directory. Return the cached directory /// if this was successful, None otherwise. Similarly, if the package id /// refers to a git repo on the local version, also check it out. /// (right now we only support git) - pub fn fetch_git(&self) -> Option { + pub fn fetch_git(local: &Path, pkgid: &PkgId) -> Option { use conditions::failed_to_create_temp_dir::cond; // We use a temporary directory because if the git clone fails, @@ -103,38 +142,35 @@ impl PkgSrc { None => cond.raise(~"Failed to create temporary directory for fetching git sources") }; - let mut local = self.root.push("src"); - local = local.push(self.id.to_str()); - - debug!("Checking whether %s exists locally. Cwd = %s, does it? %?", - self.id.path.to_str(), + debug!("Checking whether %s (path = %s) exists locally. Cwd = %s, does it? %?", + pkgid.to_str(), pkgid.path.to_str(), os::getcwd().to_str(), - os::path_exists(&self.id.path)); + os::path_exists(&pkgid.path)); - if os::path_exists(&self.id.path) { + if os::path_exists(&pkgid.path) { debug!("%s exists locally! Cloning it into %s", - self.id.path.to_str(), local.to_str()); + pkgid.path.to_str(), local.to_str()); // Ok to use local here; we know it will succeed - git_clone(&self.id.path, &local, &self.id.version); - return Some(local); + git_clone(&pkgid.path, local, &pkgid.version); + return Some(local.clone()); } - if self.id.path.components().len() < 2 { + if pkgid.path.components().len() < 2 { // If a non-URL, don't bother trying to fetch return None; } - let url = fmt!("https://%s", self.id.path.to_str()); - note(fmt!("Fetching package: git clone %s %s [version=%s]", - url, clone_target.to_str(), self.id.version.to_str())); + let url = fmt!("https://%s", pkgid.path.to_str()); + debug!("Fetching package: git clone %s %s [version=%s]", + url, clone_target.to_str(), pkgid.version.to_str()); - if git_clone_general(url, &clone_target, &self.id.version) { + if git_clone_general(url, &clone_target, &pkgid.version) { // since the operation succeeded, move clone_target to local - if !os::rename_file(&clone_target, &local) { + if !os::rename_file(&clone_target, local) { None } else { - Some(local) + Some(local.clone()) } } else { @@ -143,10 +179,11 @@ impl PkgSrc { } - // If a file named "pkg.rs" in the current directory exists, + // If a file named "pkg.rs" in the start directory exists, // return the path for it. Otherwise, None - pub fn package_script_option(&self, cwd: &Path) -> Option { - let maybe_path = cwd.push("pkg.rs"); + pub fn package_script_option(&self) -> Option { + let maybe_path = self.start_dir.push("pkg.rs"); + debug!("package_script_option: checking whether %s exists", maybe_path.to_str()); if os::path_exists(&maybe_path) { Some(maybe_path) } @@ -166,20 +203,18 @@ impl PkgSrc { for c in p.components.slice(prefix, p.components.len()).iter() { sub = sub.push(*c); } - debug!("found crate %s", sub.to_str()); + debug!("Will compile crate %s", sub.to_str()); cs.push(Crate::new(&sub)); } /// Infers crates to build. Called only in the case where there /// is no custom build logic - pub fn find_crates(&mut self, cx: &Ctx) { + pub fn find_crates(&mut self) { use conditions::missing_pkg_files::cond; - let dir = self.check_dir(cx); - debug!("Called check_dir, I'm in %s", dir.to_str()); - let prefix = dir.components.len(); - debug!("Matching against %?", self.id.short_name); - do os::walk_dir(&dir) |pth| { + let prefix = self.start_dir.components.len(); + debug!("Matching against %s", self.id.short_name); + do os::walk_dir(&self.start_dir) |pth| { let maybe_known_crate_set = match pth.filename() { Some(filename) => match filename { "lib.rs" => Some(&mut self.libs), @@ -207,7 +242,8 @@ impl PkgSrc { cond.raise(self.id.clone()); } - debug!("found %u libs, %u mains, %u tests, %u benchs", + debug!("In %s, found %u libs, %u mains, %u tests, %u benchs", + self.start_dir.to_str(), self.libs.len(), self.mains.len(), self.tests.len(), @@ -215,68 +251,93 @@ impl PkgSrc { } fn build_crates(&self, - ctx: &Ctx, - src_dir: &Path, + ctx: &BuildCtx, + exec: &mut workcache::Exec, destination_dir: &Path, crates: &[Crate], cfgs: &[~str], what: OutputType) { for crate in crates.iter() { - let path = &src_dir.push_rel(&crate.file).normalize(); - note(fmt!("build_crates: compiling %s", path.to_str())); - note(fmt!("build_crates: using as workspace %s", self.root.to_str())); + let path = self.start_dir.push_rel(&crate.file).normalize(); + debug!("build_crates: compiling %s", path.to_str()); + let path_str = path.to_str(); + let cfgs = crate.cfgs + cfgs; - let result = compile_crate(ctx, - &self.id, - path, - // compile_crate wants the destination workspace - destination_dir, - crate.flags, - crate.cfgs + cfgs, - false, - what); - if !result { - build_err::cond.raise(fmt!("build failure on %s", - path.to_str())); - } - debug!("Result of compiling %s was %?", - path.to_str(), result); + let result = { + // compile_crate should return the path of the output artifact + match compile_crate(ctx, + exec, + &self.id, + &path, + destination_dir, + crate.flags, + cfgs, + false, + what).map(|p| p.to_str()) { + Some(p) => p, + None => build_err::cond.raise(fmt!("build failure on %s", + path_str)) + + } + }; + debug!("Result of compiling %s was %s", path_str, result); } } - pub fn build(&self, ctx: &Ctx, cfgs: ~[~str]) -> Path { + /// Declare all the crate files in the package source as inputs + pub fn declare_inputs(&self, prep: &mut workcache::Prep) { + let to_do = ~[self.libs.clone(), self.mains.clone(), + self.tests.clone(), self.benchs.clone()]; + for cs in to_do.iter() { + for c in cs.iter() { + let path = self.start_dir.push_rel(&c.file).normalize(); + debug!("Declaring input: %s", path.to_str()); + prep.declare_input("file", + path.to_str(), + workcache_support::digest_file_with_date(&path.clone())); + } + } + } + + // It would be better if build returned a Path, but then Path would have to derive + // Encodable. + pub fn build(&self, exec: &mut workcache::Exec, ctx: &BuildCtx, cfgs: ~[~str]) -> ~str { use conditions::not_a_workspace::cond; // Determine the destination workspace (which depends on whether // we're using the rust_path_hack) - let destination_workspace = if is_workspace(&self.root) { - debug!("%s is indeed a workspace", self.root.to_str()); - self.root.clone() + let destination_workspace = if is_workspace(&self.workspace) { + debug!("%s is indeed a workspace", self.workspace.to_str()); + self.workspace.clone() } - else { + else { // It would be nice to have only one place in the code that checks // for the use_rust_path_hack flag... - if ctx.use_rust_path_hack { + if ctx.cx.use_rust_path_hack { let rs = default_workspace(); debug!("Using hack: %s", rs.to_str()); rs } else { cond.raise(fmt!("Package root %s is not a workspace; pass in --rust_path_hack \ - if you want to treat it as a package source", self.root.to_str())) + if you want to treat it as a package source", + self.workspace.to_str())) } }; - let dir = self.check_dir(ctx); - debug!("Building libs in %s, destination = %s", dir.to_str(), - destination_workspace.to_str()); - self.build_crates(ctx, &dir, &destination_workspace, self.libs, cfgs, Lib); + let libs = self.libs.clone(); + let mains = self.mains.clone(); + let tests = self.tests.clone(); + let benchs = self.benchs.clone(); + debug!("Building libs in %s, destination = %s", + destination_workspace.to_str(), destination_workspace.to_str()); + self.build_crates(ctx, exec, &destination_workspace, libs, cfgs, Lib); debug!("Building mains"); - self.build_crates(ctx, &dir, &destination_workspace, self.mains, cfgs, Main); + self.build_crates(ctx, exec, &destination_workspace, mains, cfgs, Main); debug!("Building tests"); - self.build_crates(ctx, &dir, &destination_workspace, self.tests, cfgs, Test); + self.build_crates(ctx, exec, &destination_workspace, tests, cfgs, Test); debug!("Building benches"); - self.build_crates(ctx, &dir, &destination_workspace, self.benchs, cfgs, Bench); - destination_workspace + self.build_crates(ctx, exec, &destination_workspace, benchs, cfgs, Bench); + destination_workspace.to_str() } } diff --git a/src/librustpkg/path_util.rs b/src/librustpkg/path_util.rs index cbe6c8f65c4..566e75df548 100644 --- a/src/librustpkg/path_util.rs +++ b/src/librustpkg/path_util.rs @@ -14,7 +14,6 @@ pub use package_id::PkgId; pub use target::{OutputType, Main, Lib, Test, Bench, Target, Build, Install}; pub use version::{Version, NoVersion, split_version_general, try_parsing_version}; pub use rustc::metadata::filesearch::rust_path; -use context::Ctx; use std::libc::consts::os::posix88::{S_IRUSR, S_IWUSR, S_IXUSR}; use std::os::mkdir_recursive; @@ -62,12 +61,7 @@ pub fn workspace_contains_package_id_(pkgid: &PkgId, workspace: &Path, let mut found = None; do os::walk_dir(&src_dir) |p| { - debug!("=> p = %s", p.to_str()); - if os::path_is_dir(p) { - debug!("p = %s, path = %s [%s]", p.to_str(), pkgid.path.to_str(), - src_dir.push_rel(&pkgid.path).to_str()); - if *p == src_dir.push_rel(&pkgid.path) || { let pf = p.filename(); do pf.iter().any |pf| { @@ -75,8 +69,6 @@ pub fn workspace_contains_package_id_(pkgid: &PkgId, workspace: &Path, match split_version_general(g, '-') { None => false, Some((ref might_match, ref vers)) => { - debug!("might_match = %s, vers = %s", *might_match, - vers.to_str()); *might_match == pkgid.short_name && (pkgid.version == *vers || pkgid.version == NoVersion) } @@ -90,32 +82,15 @@ pub fn workspace_contains_package_id_(pkgid: &PkgId, workspace: &Path, true }; - debug!(if found.is_some() { fmt!("Found %s in %s", pkgid.to_str(), workspace.to_str()) } - else { fmt!("Didn't find %s in %s", pkgid.to_str(), workspace.to_str()) }); + if found.is_some() { + debug!("Found %s in %s", pkgid.to_str(), workspace.to_str()); + } + else { + debug!("Didn't find %s in %s", pkgid.to_str(), workspace.to_str()); + } found } -/// Returns a list of possible directories -/// for 's source files in . -/// Doesn't check that any of them exist. -/// (for example, try both with and without the version) -pub fn pkgid_src_in_workspace(pkgid: &PkgId, workspace: &Path) -> ~[Path] { - let mut results = ~[]; - let result = workspace.push("src").push(fmt!("%s-%s", - pkgid.path.to_str(), pkgid.version.to_str())); - results.push(result); - results.push(workspace.push("src").push_rel(&pkgid.path)); - results -} - -/// Returns a src for pkgid that does exist -- None if none of them do -pub fn first_pkgid_src_in_workspace(pkgid: &PkgId, workspace: &Path) -> Option { - let rs = pkgid_src_in_workspace(pkgid, workspace); - do rs.iter().find |&p| { - os::path_exists(p) - }.map(|p| (**p).clone()) -} - /// Figure out what the executable name for in 's build /// directory is, and if the file exists, return it. pub fn built_executable_in_workspace(pkgid: &PkgId, workspace: &Path) -> Option { @@ -402,10 +377,7 @@ fn dir_has_file(dir: &Path, file: &str) -> bool { os::path_exists(&dir.push(file)) } -pub fn find_dir_using_rust_path_hack(cx: &Ctx, p: &PkgId) -> Option { - if !cx.use_rust_path_hack { - return None; - } +pub fn find_dir_using_rust_path_hack(p: &PkgId) -> Option { let rp = rust_path(); for dir in rp.iter() { debug!("In find_dir_using_rust_path_hack: checking dir %s", dir.to_str()); diff --git a/src/librustpkg/rustpkg.rs b/src/librustpkg/rustpkg.rs index 25a415df302..27bde2bca6d 100644 --- a/src/librustpkg/rustpkg.rs +++ b/src/librustpkg/rustpkg.rs @@ -24,8 +24,9 @@ extern mod syntax; use std::{io, os, result, run, str}; pub use std::path::Path; -use std::hashmap::HashMap; +use extra::workcache; +use extra::arc::RWArc; use rustc::driver::{driver, session}; use rustc::metadata::filesearch; use rustc::metadata::filesearch::rust_path; @@ -33,15 +34,16 @@ use extra::{getopts}; use syntax::{ast, diagnostic}; use util::*; use messages::*; -use path_util::{build_pkg_id_in_workspace, first_pkgid_src_in_workspace}; +use path_util::build_pkg_id_in_workspace; use path_util::{U_RWX, in_rust_path}; use path_util::{built_executable_in_workspace, built_library_in_workspace, default_workspace}; use path_util::{target_executable_in_workspace, target_library_in_workspace}; use source_control::is_git_dir; use workspace::{each_pkg_parent_workspace, pkg_parent_workspaces, cwd_to_workspace}; -use context::Ctx; +use context::{BuildCtx, Ctx}; use package_id::PkgId; use package_source::PkgSrc; +use workcache_support::{discover_outputs, digest_only_date}; pub mod api; mod conditions; @@ -59,6 +61,7 @@ mod target; mod tests; mod util; mod version; +pub mod workcache_support; mod workspace; pub mod usage; @@ -69,11 +72,8 @@ pub mod usage; struct PkgScript<'self> { /// Uniquely identifies this package id: &'self PkgId, - // Used to have this field: deps: ~[(~str, Option<~str>)] - // but I think it shouldn't be stored here - /// The contents of the package script: either a file path, - /// or a string containing the text of the input - input: driver::input, + /// File path for the package script + input: Path, /// The session to use *only* for compiling the custom /// build script sess: session::Session, @@ -104,7 +104,7 @@ impl<'self> PkgScript<'self> { crate_type: session::bin_crate, .. (*session::basic_options()).clone() }; - let input = driver::file_input(script); + let input = driver::file_input(script.clone()); let sess = driver::build_session(options, diagnostic::emit); let cfg = driver::build_configuration(sess); let crate = driver::phase_1_parse_input(sess, cfg.clone(), &input); @@ -115,7 +115,7 @@ impl<'self> PkgScript<'self> { PkgScript { id: id, - input: input, + input: script, sess: sess, cfg: cfg, crate: crate, @@ -127,22 +127,24 @@ impl<'self> PkgScript<'self> { /// is the command to pass to it (e.g., "build", "clean", "install") /// Returns a pair of an exit code and list of configs (obtained by /// calling the package script's configs() function if it exists - // FIXME (#4432): Use workcache to only compile the script when changed - fn run_custom(&self, sysroot: @Path) -> (~[~str], ExitCode) { + fn run_custom(&self, exec: &mut workcache::Exec, sysroot: &Path) -> (~[~str], ExitCode) { let sess = self.sess; debug!("Working directory = %s", self.build_dir.to_str()); // Collect together any user-defined commands in the package script let crate = util::ready_crate(sess, self.crate); debug!("Building output filenames with script name %s", - driver::source_name(&self.input)); + driver::source_name(&driver::file_input(self.input.clone()))); let exe = self.build_dir.push(~"pkg" + util::exe_suffix()); util::compile_crate_from_input(&self.input, + exec, &self.build_dir, sess, crate); debug!("Running program: %s %s %s", exe.to_str(), sysroot.to_str(), "install"); + // Discover the output + exec.discover_output("binary", exe.to_str(), digest_only_date(&exe)); // FIXME #7401 should support commands besides `install` let status = run::process_status(exe.to_str(), [sysroot.to_str(), ~"install"]); if status != 0 { @@ -162,47 +164,65 @@ impl<'self> PkgScript<'self> { fn hash(&self) -> ~str { self.id.hash() } - } pub trait CtxMethods { fn run(&self, cmd: &str, args: ~[~str]); fn do_cmd(&self, _cmd: &str, _pkgname: &str); + fn build_from_src(&self, pkg_src: PkgSrc); /// Returns the destination workspace - fn build(&self, workspace: &Path, pkgid: &PkgId) -> Path; + fn build(&self, exec: &mut workcache::Exec, pkg_src: PkgSrc) -> Path; fn clean(&self, workspace: &Path, id: &PkgId); fn info(&self); - fn install(&self, workspace: &Path, id: &PkgId); - fn install_no_build(&self, workspace: &Path, id: &PkgId); + fn install(&self, src: PkgSrc) -> (~[Path], ~[(~str, ~str)]); + /// Returns a list of installed files + fn install_no_build(&self, workspace: &Path, id: &PkgId) -> ~[Path]; fn prefer(&self, _id: &str, _vers: Option<~str>); fn test(&self); fn uninstall(&self, _id: &str, _vers: Option<~str>); fn unprefer(&self, _id: &str, _vers: Option<~str>); } -impl CtxMethods for Ctx { +impl CtxMethods for BuildCtx { + fn build_from_src(&self, pkg_src: PkgSrc) { + let tag = pkg_src.id.to_str(); + debug!("package source = %s", pkg_src.to_str()); + do self.workcache_cx.with_prep(tag) |prep| { + let subsrc = pkg_src.clone(); + let subself = self.clone(); + declare_package_script_dependency(prep, &subsrc); + pkg_src.declare_inputs(prep); + do prep.exec |exec| { + subself.build(exec, subsrc.clone()); + } + } + } fn run(&self, cmd: &str, args: ~[~str]) { match cmd { "build" => { if args.len() < 1 { match cwd_to_workspace() { - None if self.use_rust_path_hack => { + None if self.cx.use_rust_path_hack => { let cwd = os::getcwd(); - self.build(&cwd, &PkgId::new(cwd.components[cwd.components.len() - 1])); + let pkgid = PkgId::new(cwd.components[cwd.components.len() - 1]); + self.build_from_src(PkgSrc::new(cwd, true, pkgid)); } None => { usage::build(); return; } - Some((ws, pkgid)) => { self.build(&ws, &pkgid); } + Some((ws, pkgid)) => { + self.build_from_src(PkgSrc::new(ws, false, pkgid)); + } } } else { // The package id is presumed to be the first command-line // argument let pkgid = PkgId::new(args[0].clone()); - do each_pkg_parent_workspace(self, &pkgid) |workspace| { + do each_pkg_parent_workspace(&self.cx, &pkgid) |workspace| { debug!("found pkg %s in workspace %s, trying to build", pkgid.to_str(), workspace.to_str()); - self.build(workspace, &pkgid); + let pkg_src = PkgSrc::new(workspace.clone(), false, pkgid.clone()); + self.build_from_src(pkg_src); true }; } @@ -236,35 +256,40 @@ impl CtxMethods for Ctx { self.info(); } "install" => { - if args.len() < 1 { + if args.len() < 1 { match cwd_to_workspace() { - None if self.use_rust_path_hack => { + None if self.cx.use_rust_path_hack => { let cwd = os::getcwd(); - self.install(&cwd, - &PkgId::new(cwd.components[cwd.components.len() - 1])); + let inferred_pkgid = + PkgId::new(cwd.components[cwd.components.len() - 1]); + self.install(PkgSrc::new(cwd, true, inferred_pkgid)); } None => { usage::install(); return; } - Some((ws, pkgid)) => self.install(&ws, &pkgid), - } + Some((ws, pkgid)) => { + let pkg_src = PkgSrc::new(ws, false, pkgid); + self.install(pkg_src); + } + } } else { // The package id is presumed to be the first command-line // argument let pkgid = PkgId::new(args[0]); - let workspaces = pkg_parent_workspaces(self, &pkgid); + let workspaces = pkg_parent_workspaces(&self.cx, &pkgid); debug!("package ID = %s, found it in %? workspaces", pkgid.to_str(), workspaces.len()); if workspaces.is_empty() { let rp = rust_path(); assert!(!rp.is_empty()); - let src = PkgSrc::new(&rp[0], &pkgid); - src.fetch_git(); - self.install(&rp[0], &pkgid); + let src = PkgSrc::new(rp[0].clone(), false, pkgid.clone()); + self.install(src); } else { - do each_pkg_parent_workspace(self, &pkgid) |workspace| { - self.install(workspace, &pkgid); - true + for workspace in workspaces.iter() { + let src = PkgSrc::new(workspace.clone(), + self.cx.use_rust_path_hack, + pkgid.clone()); + self.install(src); }; } } @@ -299,7 +324,7 @@ impl CtxMethods for Ctx { else { let rp = rust_path(); assert!(!rp.is_empty()); - do each_pkg_parent_workspace(self, &pkgid) |workspace| { + do each_pkg_parent_workspace(&self.cx, &pkgid) |workspace| { path_util::uninstall_package_from(workspace, &pkgid); note(fmt!("Uninstalled package %s (was installed in %s)", pkgid.to_str(), workspace.to_str())); @@ -325,44 +350,44 @@ impl CtxMethods for Ctx { /// Returns the destination workspace /// In the case of a custom build, we don't know, so we just return the source workspace - fn build(&self, workspace: &Path, pkgid: &PkgId) -> Path { + fn build(&self, exec: &mut workcache::Exec, pkg_src: PkgSrc) -> Path { + + let pkg_src = &mut pkg_src.clone(); // :-o + let workspace = pkg_src.workspace.clone(); + let pkgid = pkg_src.id.clone(); + debug!("build: workspace = %s (in Rust path? %? is git dir? %? \ - pkgid = %s", workspace.to_str(), - in_rust_path(workspace), is_git_dir(&workspace.push_rel(&pkgid.path)), - pkgid.to_str()); - let src_dir = first_pkgid_src_in_workspace(pkgid, workspace); + pkgid = %s pkgsrc start_dir = %s", workspace.to_str(), + in_rust_path(&workspace), is_git_dir(&workspace.push_rel(&pkgid.path)), + pkgid.to_str(), pkg_src.start_dir.to_str()); // If workspace isn't in the RUST_PATH, and it's a git repo, // then clone it into the first entry in RUST_PATH, and repeat - debug!("%? %? %s", in_rust_path(workspace), - is_git_dir(&workspace.push_rel(&pkgid.path)), - workspace.to_str()); - if !in_rust_path(workspace) && is_git_dir(&workspace.push_rel(&pkgid.path)) { + if !in_rust_path(&workspace) && is_git_dir(&workspace.push_rel(&pkgid.path)) { let out_dir = default_workspace().push("src").push_rel(&pkgid.path); source_control::git_clone(&workspace.push_rel(&pkgid.path), &out_dir, &pkgid.version); let default_ws = default_workspace(); debug!("Calling build recursively with %? and %?", default_ws.to_str(), pkgid.to_str()); - return self.build(&default_ws, pkgid); + return self.build(exec, PkgSrc::new(default_ws, false, pkgid.clone())); } - // Create the package source - let mut src = PkgSrc::new(workspace, pkgid); - debug!("Package src = %?", src); - // Is there custom build logic? If so, use it - let pkg_src_dir = src_dir; let mut custom = false; - debug!("Package source directory = %?", pkg_src_dir); - let cfgs = match pkg_src_dir.chain_ref(|p| src.package_script_option(p)) { + debug!("Package source directory = %s", pkg_src.to_str()); + let opt = pkg_src.package_script_option(); + debug!("Calling pkg_script_option on %?", opt); + let cfgs = match pkg_src.package_script_option() { Some(package_script_path) => { - let sysroot = self.sysroot_to_use().expect("custom build needs a sysroot"); - let pscript = PkgScript::parse(sysroot, - package_script_path, - workspace, - pkgid); - let (cfgs, hook_result) = pscript.run_custom(sysroot); + let sysroot = self.sysroot_to_use(); + let (cfgs, hook_result) = { + let pscript = PkgScript::parse(@sysroot.clone(), + package_script_path.clone(), + &workspace.clone(), + &pkgid); + pscript.run_custom(exec, &sysroot) + }; debug!("Command return code = %?", hook_result); if hook_result != 0 { fail!("Error running custom build command") @@ -381,9 +406,10 @@ impl CtxMethods for Ctx { // the build already. Otherwise... if !custom { // Find crates inside the workspace - src.find_crates(self); + pkg_src.find_crates(); // Build it! - src.build(self, cfgs) + let rs_path = pkg_src.build(exec, self, cfgs); + Path(rs_path) } else { // Just return the source workspace @@ -412,20 +438,47 @@ impl CtxMethods for Ctx { fail!("info not yet implemented"); } - fn install(&self, workspace: &Path, id: &PkgId) { - // Also should use workcache to not build if not necessary. - let destination_workspace = self.build(workspace, id); - // See #7402: This still isn't quite right yet; we want to - // install to the first workspace in the RUST_PATH if there's - // a non-default RUST_PATH. This code installs to the same - // workspace the package was built in. - debug!("install: destination workspace = %s, id = %s", - destination_workspace.to_str(), id.to_str()); - self.install_no_build(&destination_workspace, id); + /// Returns a pair. First component is a list of installed paths, + /// second is a list of declared and discovered inputs + fn install(&self, pkg_src: PkgSrc) -> (~[Path], ~[(~str, ~str)]) { + let id = &pkg_src.id; + + let installed_files = RWArc::new(~[]); + let inputs = RWArc::new(~[]); + // FIXME #7402: Use RUST_PATH to determine target dir + let f: &fn(&mut workcache::Prep) = |prep| { + let sub_inputs = inputs.clone(); + let sub_files = installed_files.clone(); + let subsrc = pkg_src.clone(); + let subself = self.clone(); + let id_str = id.to_str(); + let sub_id = id.clone(); + sub_inputs.write(|r| *r = prep.lookup_declared_inputs().map(|v| + { (~"file", (*v).clone()) })); + do prep.exec |exec| { + let destination_workspace = subself.build(exec, subsrc.clone()).to_str(); + // See #7402: This still isn't quite right yet; we want to + // install to the first workspace in the RUST_PATH if there's + // a non-default RUST_PATH. This code installs to the same + // workspace the package was built in. + debug!("install: destination workspace = %s, id = %s", + destination_workspace, id_str); + let result = subself.install_no_build(&Path(destination_workspace), &sub_id); + debug!("install: id = %s, about to call discover_outputs, %?", + id_str, result.to_str()); + + discover_outputs(exec, result.clone()); + sub_files.write(|r| { *r = result.clone(); }); + sub_inputs.write(|r| { *r = *r + exec.lookup_discovered_inputs() }); + } + }; + self.workcache_cx.with_prep(id.to_str(), |p| pkg_src.declare_inputs(p)); + self.workcache_cx.with_prep(id.to_str(), f); + (installed_files.unwrap(), inputs.unwrap()) } - fn install_no_build(&self, workspace: &Path, id: &PkgId) { + fn install_no_build(&self, workspace: &Path, id: &PkgId) -> ~[Path] { use conditions::copy_failed::cond; // Now copy stuff into the install dirs @@ -439,12 +492,15 @@ impl CtxMethods for Ctx { target_exec.to_str(), target_lib, maybe_executable, maybe_library); + let mut outputs = ~[]; + for exec in maybe_executable.iter() { debug!("Copying: %s -> %s", exec.to_str(), target_exec.to_str()); if !(os::mkdir_recursive(&target_exec.dir_path(), U_RWX) && os::copy_file(exec, &target_exec)) { cond.raise(((*exec).clone(), target_exec.clone())); } + outputs.push(target_exec.clone()); } for lib in maybe_library.iter() { let target_lib = target_lib.clone().expect(fmt!("I built %s but apparently \ @@ -455,7 +511,9 @@ impl CtxMethods for Ctx { os::copy_file(lib, &target_lib)) { cond.raise(((*lib).clone(), target_lib.clone())); } + outputs.push(target_lib.clone()); } + outputs } fn prefer(&self, _id: &str, _vers: Option<~str>) { @@ -476,7 +534,6 @@ impl CtxMethods for Ctx { } } - pub fn main() { io::println("WARNING: The Rust package manager is experimental and may be unstable"); let args = os::args(); @@ -485,7 +542,6 @@ pub fn main() { pub fn main_args(args: &[~str]) { let opts = ~[getopts::optflag("h"), getopts::optflag("help"), - getopts::optflag("j"), getopts::optflag("json"), getopts::optmulti("c"), getopts::optmulti("cfg"), getopts::optflag("v"), getopts::optflag("version"), getopts::optflag("r"), getopts::optflag("rust-path-hack")]; @@ -499,8 +555,6 @@ pub fn main_args(args: &[~str]) { }; let help = getopts::opt_present(matches, "h") || getopts::opt_present(matches, "help"); - let json = getopts::opt_present(matches, "j") || - getopts::opt_present(matches, "json"); if getopts::opt_present(matches, "v") || getopts::opt_present(matches, "version") { @@ -512,7 +566,6 @@ pub fn main_args(args: &[~str]) { getopts::opt_present(matches, "rust-path-hack"); let mut args = matches.free.clone(); - args.shift(); if (args.len() < 1) { @@ -553,13 +606,15 @@ pub fn main_args(args: &[~str]) { // I had to add this type annotation to get the code to typecheck let mut remaining_args: ~[~str] = remaining_args.map(|s| (*s).clone()).collect(); remaining_args.shift(); - let sroot = Some(@filesearch::get_or_default_sysroot()); - debug!("Using sysroot: %?", sroot); - Ctx { - use_rust_path_hack: use_rust_path_hack, - sysroot_opt: sroot, // Currently, only tests override this - json: json, - dep_cache: @mut HashMap::new() + let sroot = filesearch::get_or_default_sysroot(); + debug!("Using sysroot: %s", sroot.to_str()); + debug!("Will store workcache in %s", default_workspace().to_str()); + BuildCtx { + cx: Ctx { + use_rust_path_hack: use_rust_path_hack, + sysroot_opt: sroot, // Currently, only tests override this + }, + workcache_cx: api::default_ctxt(default_workspace()).workcache_cx // ??? }.run(*cmd, remaining_args) } @@ -581,3 +636,11 @@ pub fn work_dir() -> Path { pub fn src_dir() -> Path { os::getcwd() } + +fn declare_package_script_dependency(prep: &mut workcache::Prep, pkg_src: &PkgSrc) { + match pkg_src.package_script_option() { + Some(ref p) => prep.declare_input("file", p.to_str(), + workcache_support::digest_file_with_date(p)), + None => () + } +} diff --git a/src/librustpkg/search.rs b/src/librustpkg/search.rs index 9862f870bca..e5e2a0dbd71 100644 --- a/src/librustpkg/search.rs +++ b/src/librustpkg/search.rs @@ -11,16 +11,6 @@ use path_util::{installed_library_in_workspace, rust_path}; use version::Version; -/// If a library with path `p` matching pkg_id's name exists under sroot_opt, -/// return Some(p). Return None if there's no such path or if sroot_opt is None. -pub fn find_library_in_search_path(sroot_opt: Option<@Path>, short_name: &str) -> Option { - do sroot_opt.chain |sroot| { - debug!("Will search for a library with short name %s in \ - %s", short_name, (sroot.push("lib")).to_str()); - installed_library_in_workspace(short_name, sroot) - } -} - /// If some workspace `p` in the RUST_PATH contains a package matching short_name, /// return Some(p) (returns the first one of there are multiple matches.) Return /// None if there's no such path. diff --git a/src/librustpkg/tests.rs b/src/librustpkg/tests.rs index a86f299276a..67d8732cf32 100644 --- a/src/librustpkg/tests.rs +++ b/src/librustpkg/tests.rs @@ -10,10 +10,13 @@ // rustpkg unit tests -use context::Ctx; -use std::hashmap::HashMap; -use std::{io, libc, os, run, str}; +use context::{BuildCtx, Ctx}; +use std::{io, libc, os, run, str, task}; +use extra::arc::Arc; +use extra::arc::RWArc; use extra::tempfile::mkdtemp; +use extra::workcache::{Context, Database, Logger}; +use extra::treemap::TreeMap; use std::run::ProcessOutput; use installed_packages::list_installed_packages; use package_id::{PkgId}; @@ -26,18 +29,23 @@ use path_util::{target_executable_in_workspace, target_test_in_workspace, use rustc::metadata::filesearch::rust_path; use rustc::driver::driver::host_triple; use target::*; +use package_source::PkgSrc; /// Returns the last-modified date as an Option fn datestamp(p: &Path) -> Option { p.stat().map(|stat| stat.st_mtime) } -fn fake_ctxt(sysroot_opt: Option<@Path>) -> Ctx { - Ctx { - use_rust_path_hack: false, - sysroot_opt: sysroot_opt, - json: false, - dep_cache: @mut HashMap::new() +fn fake_ctxt(sysroot_opt: Path, workspace: &Path) -> BuildCtx { + let bcx = Context::new(RWArc::new(Database::new(workspace.push("rustpkg_db.json"))), + RWArc::new(Logger::new()), + Arc::new(TreeMap::new())); + BuildCtx { + workcache_cx: bcx, + cx: Ctx { + use_rust_path_hack: false, + sysroot_opt: sysroot_opt + } } } @@ -388,7 +396,7 @@ fn lib_output_file_name(workspace: &Path, parent: &str, short_name: &str) -> Pat } fn output_file_name(workspace: &Path, short_name: &str) -> Path { - workspace.push(fmt!("%s%s", short_name, os::EXE_SUFFIX)) + workspace.push("build").push(short_name).push(fmt!("%s%s", short_name, os::EXE_SUFFIX)) } fn touch_source_file(workspace: &Path, pkgid: &PkgId) { @@ -401,12 +409,11 @@ fn touch_source_file(workspace: &Path, pkgid: &PkgId) { if run::process_output("touch", [p.to_str()]).status != 0 { let _ = cond.raise((pkg_src_dir.clone(), ~"Bad path")); } - break; } } } -/// Add a blank line at the end +/// Add a comment at the end fn frob_source_file(workspace: &Path, pkgid: &PkgId) { use conditions::bad_path::cond; let pkg_src_dir = workspace.push("src").push(pkgid.to_str()); @@ -423,7 +430,7 @@ fn frob_source_file(workspace: &Path, pkgid: &PkgId) { let w = io::file_writer(p, &[io::Append]); match w { Err(s) => { let _ = cond.raise((p.clone(), fmt!("Bad path: %s", s))); } - Ok(w) => w.write_line("") + Ok(w) => w.write_line("/* hi */") } } None => fail!(fmt!("frob_source_file failed to find a source file in %s", @@ -450,12 +457,13 @@ fn test_install_valid() { let sysroot = test_sysroot(); debug!("sysroot = %s", sysroot.to_str()); - let ctxt = fake_ctxt(Some(@sysroot)); let temp_pkg_id = fake_pkg(); let temp_workspace = mk_temp_workspace(&temp_pkg_id.path, &NoVersion).pop().pop(); + let ctxt = fake_ctxt(sysroot, &temp_workspace); debug!("temp_workspace = %s", temp_workspace.to_str()); // should have test, bench, lib, and main - ctxt.install(&temp_workspace, &temp_pkg_id); + let src = PkgSrc::new(temp_workspace.clone(), false, temp_pkg_id.clone()); + ctxt.install(src); // Check that all files exist let exec = target_executable_in_workspace(&temp_pkg_id, &temp_workspace); debug!("exec = %s", exec.to_str()); @@ -476,32 +484,19 @@ fn test_install_valid() { #[test] fn test_install_invalid() { - use conditions::nonexistent_package::cond; - use cond1 = conditions::missing_pkg_files::cond; - use cond2 = conditions::not_a_workspace::cond; - - let ctxt = fake_ctxt(None); + let sysroot = test_sysroot(); let pkgid = fake_pkg(); let temp_workspace = mkdtemp(&os::tmpdir(), "test").expect("couldn't create temp dir"); - let mut error_occurred = false; - let mut error1_occurred = false; - let mut error2_occurred = false; - do cond1.trap(|_| { - error1_occurred = true; - }).inside { - do cond.trap(|_| { - error_occurred = true; - temp_workspace.clone() - }).inside { - do cond2.trap(|_| { - error2_occurred = true; - temp_workspace.clone() - }).inside { - ctxt.install(&temp_workspace, &pkgid); - } - } - } - assert!(error_occurred && error1_occurred && error2_occurred); + let ctxt = fake_ctxt(sysroot, &temp_workspace); + + // Uses task::try because of #9001 + let result = do task::try { + let pkg_src = PkgSrc::new(temp_workspace.clone(), false, pkgid.clone()); + ctxt.install(pkg_src); + }; + // Not the best test -- doesn't test that we failed in the right way. + // Best we can do for now. + assert!(result == Err(())); } // Tests above should (maybe) be converted to shell out to rustpkg, too @@ -898,7 +893,6 @@ fn install_check_duplicates() { } #[test] -#[ignore(reason = "Workcache not yet implemented -- see #7075")] fn no_rebuilding() { let p_id = PkgId::new("foo"); let workspace = create_local_package(&p_id); @@ -912,24 +906,28 @@ fn no_rebuilding() { } #[test] -#[ignore(reason = "Workcache not yet implemented -- see #7075")] fn no_rebuilding_dep() { let p_id = PkgId::new("foo"); let dep_id = PkgId::new("bar"); let workspace = create_local_package_with_dep(&p_id, &dep_id); command_line_test([~"build", ~"foo"], &workspace); - let bar_date = datestamp(&lib_output_file_name(&workspace, + let bar_date_1 = datestamp(&lib_output_file_name(&workspace, ".rust", "bar")); - let foo_date = datestamp(&output_file_name(&workspace, "foo")); - assert!(bar_date < foo_date); + let foo_date_1 = datestamp(&output_file_name(&workspace, "foo")); + + frob_source_file(&workspace, &p_id); + command_line_test([~"build", ~"foo"], &workspace); + let bar_date_2 = datestamp(&lib_output_file_name(&workspace, + ".rust", + "bar")); + let foo_date_2 = datestamp(&output_file_name(&workspace, "foo")); + assert_eq!(bar_date_1, bar_date_2); + assert!(foo_date_1 < foo_date_2); + assert!(foo_date_1 > bar_date_1); } -// n.b. The following two tests are ignored; they worked "accidentally" before, -// when the behavior was "always rebuild libraries" (now it's "never rebuild -// libraries if they already exist"). They can be un-ignored once #7075 is done. #[test] -#[ignore(reason = "Workcache not yet implemented -- see #7075")] fn do_rebuild_dep_dates_change() { let p_id = PkgId::new("foo"); let dep_id = PkgId::new("bar"); @@ -946,7 +944,6 @@ fn do_rebuild_dep_dates_change() { } #[test] -#[ignore(reason = "Workcache not yet implemented -- see #7075")] fn do_rebuild_dep_only_contents_change() { let p_id = PkgId::new("foo"); let dep_id = PkgId::new("bar"); diff --git a/src/librustpkg/util.rs b/src/librustpkg/util.rs index 1b06f78abb9..769b1bdf25d 100644 --- a/src/librustpkg/util.rs +++ b/src/librustpkg/util.rs @@ -9,6 +9,7 @@ // except according to those terms. use std::os; +use extra::workcache; use rustc::driver::{driver, session}; use extra::getopts::groups::getopts; use syntax::ast_util::*; @@ -18,12 +19,13 @@ use syntax::{ast, attr, codemap, diagnostic, fold}; use syntax::attr::AttrMetaMethods; use rustc::back::link::output_type_exe; use rustc::driver::session::{lib_crate, bin_crate}; -use context::{Ctx, in_target}; +use context::{in_target, BuildCtx}; use package_id::PkgId; -use search::{find_library_in_search_path, find_installed_library_in_rust_path}; -use path_util::{target_library_in_workspace, U_RWX}; +use package_source::PkgSrc; +use path_util::{installed_library_in_workspace, U_RWX}; + pub use target::{OutputType, Main, Lib, Bench, Test}; -use version::NoVersion; +use workcache_support::{digest_file_with_date, digest_only_date}; // It would be nice to have the list of commands in just one place -- for example, // you could update the match in rustpkg.rc but forget to update this list. I think @@ -151,15 +153,15 @@ pub fn ready_crate(sess: session::Session, @fold.fold_crate(crate) } -// FIXME (#4432): Use workcache to only compile when needed -pub fn compile_input(ctxt: &Ctx, +pub fn compile_input(ctxt: &BuildCtx, + exec: &mut workcache::Exec, pkg_id: &PkgId, in_file: &Path, workspace: &Path, flags: &[~str], cfgs: &[~str], opt: bool, - what: OutputType) -> bool { + what: OutputType) -> Option { assert!(in_file.components.len() > 1); let input = driver::file_input((*in_file).clone()); @@ -173,7 +175,7 @@ pub fn compile_input(ctxt: &Ctx, debug!("flags: %s", flags.connect(" ")); debug!("cfgs: %s", cfgs.connect(" ")); - debug!("out_dir = %s", out_dir.to_str()); + debug!("compile_input's sysroot = %s", ctxt.sysroot_opt().to_str()); let crate_type = match what { Lib => lib_crate, @@ -191,19 +193,19 @@ pub fn compile_input(ctxt: &Ctx, driver::optgroups()).unwrap(); // Hack so that rustpkg can run either out of a rustc target dir, // or the host dir - let sysroot_to_use = if !in_target(ctxt.sysroot_opt) { - ctxt.sysroot_opt + let sysroot_to_use = @if !in_target(&ctxt.sysroot_opt()) { + ctxt.sysroot_opt() } else { - ctxt.sysroot_opt.map(|p| { @p.pop().pop().pop() }) + ctxt.sysroot_opt().pop().pop().pop() }; - debug!("compile_input's sysroot = %?", ctxt.sysroot_opt_str()); - debug!("sysroot_to_use = %?", sysroot_to_use); + debug!("compile_input's sysroot = %s", ctxt.sysroot_opt().to_str()); + debug!("sysroot_to_use = %s", sysroot_to_use.to_str()); let options = @session::options { crate_type: crate_type, optimize: if opt { session::Aggressive } else { session::No }, test: what == Test || what == Bench, - maybe_sysroot: sysroot_to_use, + maybe_sysroot: Some(sysroot_to_use), addl_lib_search_paths: @mut (~[out_dir.clone()]), // output_type should be conditional output_type: output_type_exe, // Use this to get a library? That's weird @@ -228,11 +230,11 @@ pub fn compile_input(ctxt: &Ctx, // `extern mod` directives. let cfg = driver::build_configuration(sess); let mut crate = driver::phase_1_parse_input(sess, cfg.clone(), &input); - crate = driver::phase_2_configure_and_expand(sess, cfg, crate); + crate = driver::phase_2_configure_and_expand(sess, cfg.clone(), crate); // Not really right. Should search other workspaces too, and the installed // database (which doesn't exist yet) - find_and_install_dependencies(ctxt, sess, workspace, crate, + find_and_install_dependencies(ctxt, sess, exec, workspace, crate, |p| { debug!("a dependency: %s", p.to_str()); // Pass the directory containing a dependency @@ -269,8 +271,7 @@ pub fn compile_input(ctxt: &Ctx, debug!("calling compile_crate_from_input, workspace = %s, building_library = %?", out_dir.to_str(), sess.building_library); - compile_crate_from_input(&input, &out_dir, sess, crate); - true + compile_crate_from_input(in_file, exec, &out_dir, sess, crate) } // Should use workcache to avoid recompiling when not necessary @@ -278,17 +279,19 @@ pub fn compile_input(ctxt: &Ctx, // If crate_opt is present, then finish compilation. If it's None, then // call compile_upto and return the crate // also, too many arguments -pub fn compile_crate_from_input(input: &driver::input, +pub fn compile_crate_from_input(input: &Path, + exec: &mut workcache::Exec, // should be of the form /build/ out_dir: &Path, sess: session::Session, - crate: @ast::Crate) { + crate: @ast::Crate) -> Option { debug!("Calling build_output_filenames with %s, building library? %?", out_dir.to_str(), sess.building_library); // bad copy debug!("out_dir = %s", out_dir.to_str()); - let outputs = driver::build_output_filenames(input, &Some(out_dir.clone()), &None, + let outputs = driver::build_output_filenames(&driver::file_input(input.clone()), + &Some(out_dir.clone()), &None, crate.attrs, sess); debug!("Outputs are out_filename: %s and obj_filename: %s and output type = %?", @@ -304,8 +307,13 @@ pub fn compile_crate_from_input(input: &driver::input, &analysis, outputs); driver::phase_5_run_llvm_passes(sess, &translation, outputs); - if driver::stop_after_phase_5(sess) { return; } + if driver::stop_after_phase_5(sess) { return Some(outputs.out_filename); } driver::phase_6_link_output(sess, &translation, outputs); + + // Register dependency on the source file + exec.discover_input("file", input.to_str(), digest_file_with_date(input)); + + Some(outputs.out_filename) } #[cfg(windows)] @@ -318,76 +326,90 @@ pub fn exe_suffix() -> ~str { ~".exe" } pub fn exe_suffix() -> ~str { ~"" } // Called by build_crates -// FIXME (#4432): Use workcache to only compile when needed -pub fn compile_crate(ctxt: &Ctx, pkg_id: &PkgId, +pub fn compile_crate(ctxt: &BuildCtx, + exec: &mut workcache::Exec, + pkg_id: &PkgId, crate: &Path, workspace: &Path, flags: &[~str], cfgs: &[~str], opt: bool, - what: OutputType) -> bool { + what: OutputType) -> Option { debug!("compile_crate: crate=%s, workspace=%s", crate.to_str(), workspace.to_str()); debug!("compile_crate: short_name = %s, flags =...", pkg_id.to_str()); for fl in flags.iter() { debug!("+++ %s", *fl); } - compile_input(ctxt, pkg_id, crate, workspace, flags, cfgs, opt, what) + compile_input(ctxt, exec, pkg_id, crate, workspace, flags, cfgs, opt, what) } /// Collect all `extern mod` directives in `c`, then /// try to install their targets, failing if any target /// can't be found. -pub fn find_and_install_dependencies(ctxt: &Ctx, +pub fn find_and_install_dependencies(ctxt: &BuildCtx, sess: session::Session, + exec: &mut workcache::Exec, workspace: &Path, c: &ast::Crate, save: @fn(Path) ) { - // :-( - debug!("In find_and_install_dependencies..."); - let my_workspace = (*workspace).clone(); - let my_ctxt = *ctxt; - do c.each_view_item() |vi: &ast::view_item| { + debug!("Finding and installing dependencies..."); + do c.each_view_item |vi| { debug!("A view item!"); match vi.node { // ignore metadata, I guess ast::view_item_extern_mod(lib_ident, path_opt, _, _) => { - match my_ctxt.sysroot_opt { - Some(ref x) => debug!("*** sysroot: %s", x.to_str()), - None => debug!("No sysroot given") - }; - let lib_name = match path_opt { // ??? + let lib_name = match path_opt { Some(p) => p, None => sess.str_of(lib_ident) }; - match find_library_in_search_path(my_ctxt.sysroot_opt, lib_name) { - Some(installed_path) => { + match installed_library_in_workspace(lib_name, &ctxt.sysroot_opt()) { + Some(ref installed_path) => { debug!("It exists: %s", installed_path.to_str()); + // Say that [path for c] has a discovered dependency on + // installed_path + // For binary files, we only hash the datestamp, not the contents. + // I'm not sure what the right thing is. + // Now we know that this crate has a discovered dependency on + // installed_path + exec.discover_input("binary", installed_path.to_str(), + digest_only_date(installed_path)); } None => { // FIXME #8711: need to parse version out of path_opt - match find_installed_library_in_rust_path(lib_name, &NoVersion) { - Some(installed_path) => { - debug!("Found library %s, not rebuilding it", - installed_path.to_str()); - // Once workcache is implemented, we'll actually check - // whether or not the library at installed_path is fresh - save(installed_path.pop()); + debug!("Trying to install library %s, rebuilding it", + lib_name.to_str()); + // Try to install it + let pkg_id = PkgId::new(lib_name); + let (outputs_disc, inputs_disc) = + ctxt.install(PkgSrc::new(workspace.clone(), false, pkg_id)); + debug!("Installed %s, returned %? dependencies and \ + %? transitive dependencies", + lib_name, outputs_disc.len(), inputs_disc.len()); + for dep in outputs_disc.iter() { + debug!("Discovering a binary input: %s", dep.to_str()); + exec.discover_input("binary", dep.to_str(), + digest_only_date(dep)); + } + for &(ref what, ref dep) in inputs_disc.iter() { + if *what == ~"file" { + exec.discover_input(*what, *dep, + digest_file_with_date(&Path(*dep))); } - None => { - debug!("Trying to install library %s, rebuilding it", - lib_name.to_str()); - // Try to install it - let pkg_id = PkgId::new(lib_name); - my_ctxt.install(&my_workspace, &pkg_id); - // Also, add an additional search path - debug!("let installed_path...") - let installed_path = target_library_in_workspace(&pkg_id, - &my_workspace).pop(); - debug!("Great, I installed %s, and it's in %s", - lib_name, installed_path.to_str()); - save(installed_path); - } + else if *what == ~"binary" { + exec.discover_input(*what, *dep, + digest_only_date(&Path(*dep))); + } + else { + fail!("Bad kind: %s", *what); + } + } + // Also, add an additional search path + let installed_library = + installed_library_in_workspace(lib_name, workspace) + .expect( fmt!("rustpkg failed to install dependency %s", + lib_name)); + let install_dir = installed_library.pop(); + debug!("Installed %s into %s", lib_name, install_dir.to_str()); + save(install_dir); } - } - } - } + }} // Ignore `use`s _ => () } diff --git a/src/librustpkg/version.rs b/src/librustpkg/version.rs index d408177572f..c2d87ddeb82 100644 --- a/src/librustpkg/version.rs +++ b/src/librustpkg/version.rs @@ -99,12 +99,14 @@ pub fn try_getting_local_version(local_path: &Path) -> Option { let rustpath = rust_path(); for rp in rustpath.iter() { let local_path = rp.push_rel(local_path); - debug!("in try_getting_local_version"); + let git_dir = local_path.push(".git"); + if !os::path_is_dir(&git_dir) { + loop; + } let outp = run::process_output("git", - [fmt!("--git-dir=%s", local_path.push(".git").to_str()), - ~"tag", ~"-l"]); + [fmt!("--git-dir=%s", git_dir.to_str()), ~"tag", ~"-l"]); - debug!("git --git-dir=%s tag -l ~~~> %?", local_path.push(".git").to_str(), outp.status); + debug!("git --git-dir=%s tag -l ~~~> %?", git_dir.to_str(), outp.status); if outp.status != 0 { loop; @@ -129,9 +131,7 @@ pub fn try_getting_local_version(local_path: &Path) -> Option { /// and the most recent tag in that repo denotes a version, return it; /// otherwise, `None` pub fn try_getting_version(remote_path: &Path) -> Option { - debug!("try_getting_version: %s", remote_path.to_str()); if is_url_like(remote_path) { - debug!("Trying to fetch its sources.."); let tmp_dir = mkdtemp(&os::tmpdir(), "test").expect("try_getting_version: couldn't create temp dir"); debug!("(to get version) executing {git clone https://%s %s}", @@ -218,14 +218,11 @@ pub fn split_version<'a>(s: &'a str) -> Option<(&'a str, Version)> { pub fn split_version_general<'a>(s: &'a str, sep: char) -> Option<(&'a str, Version)> { match s.rfind(sep) { Some(i) => { - debug!("in %s, i = %?", s, i); let path = s.slice(0, i); - debug!("path = %s", path); // n.b. for now, assuming an exact revision is intended, not a SemVer Some((path, ExactRevision(s.slice(i + 1, s.len()).to_owned()))) } None => { - debug!("%s doesn't look like an explicit-version thing", s); None } } diff --git a/src/librustpkg/workcache_support.rs b/src/librustpkg/workcache_support.rs new file mode 100644 index 00000000000..e2416782d98 --- /dev/null +++ b/src/librustpkg/workcache_support.rs @@ -0,0 +1,58 @@ +// Copyright 2013 The Rust Project Developers. See the COPYRIGHT +// file at the top-level directory of this distribution and at +// http://rust-lang.org/COPYRIGHT. +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +use extra::sha1::Sha1; +use extra::digest::Digest; +use extra::workcache; +use std::io; + +/// Hashes the file contents along with the last-modified time +pub fn digest_file_with_date(path: &Path) -> ~str { + use conditions::bad_path::cond; + use cond1 = conditions::bad_stat::cond; + + let mut sha = ~Sha1::new(); + let s = io::read_whole_file_str(path); + match s { + Ok(s) => { + (*sha).input_str(s); + let st = match path.stat() { + Some(st) => st, + None => cond1.raise((path.clone(), fmt!("Couldn't get file access time"))) + }; + (*sha).input_str(st.st_mtime.to_str()); + (*sha).result_str() + } + Err(e) => cond.raise((path.clone(), fmt!("Couldn't read file: %s", e))).to_str() + } +} + +/// Hashes only the last-modified time +pub fn digest_only_date(path: &Path) -> ~str { + use cond = conditions::bad_stat::cond; + + let mut sha = ~Sha1::new(); + let st = match path.stat() { + Some(st) => st, + None => cond.raise((path.clone(), fmt!("Couldn't get file access time"))) + }; + (*sha).input_str(st.st_mtime.to_str()); + (*sha).result_str() +} + +/// Adds multiple discovered outputs +pub fn discover_outputs(e: &mut workcache::Exec, outputs: ~[Path]) { + debug!("Discovering %? outputs", outputs.len()); + for p in outputs.iter() { + debug!("Discovering output! %s", p.to_str()); + // For now, assume that all discovered outputs are binaries + e.discover_output("binary", p.to_str(), digest_only_date(p)); + } +} diff --git a/src/librustpkg/workspace.rs b/src/librustpkg/workspace.rs index 5ad2dfd6d2f..cb947b0036c 100644 --- a/src/librustpkg/workspace.rs +++ b/src/librustpkg/workspace.rs @@ -43,7 +43,7 @@ pub fn pkg_parent_workspaces(cx: &Ctx, pkgid: &PkgId) -> ~[Path] { .filter(|ws| workspace_contains_package_id(pkgid, ws)) .collect(); if cx.use_rust_path_hack { - rs + option_to_vec(find_dir_using_rust_path_hack(cx, pkgid)) + rs + option_to_vec(find_dir_using_rust_path_hack(pkgid)) } else { rs diff --git a/src/libsyntax/ast_util.rs b/src/libsyntax/ast_util.rs index ee898c182e0..dcf655871df 100644 --- a/src/libsyntax/ast_util.rs +++ b/src/libsyntax/ast_util.rs @@ -14,7 +14,7 @@ use ast_util; use codemap::{Span, dummy_sp}; use opt_vec; use parse::token; -use visit::{SimpleVisitor, SimpleVisitorVisitor, Visitor}; +use visit::{SimpleVisitor, Visitor}; use visit; use std::hashmap::HashMap; @@ -684,94 +684,25 @@ pub fn walk_pat(pat: @Pat, it: &fn(@Pat) -> bool) -> bool { } pub trait EachViewItem { - fn each_view_item(&self, f: @fn(&ast::view_item) -> bool) -> bool; + fn each_view_item(&self, f: &fn(&ast::view_item) -> bool) -> bool; } -struct EachViewItemData { - callback: @fn(&ast::view_item) -> bool, +struct EachViewItemData<'self> { + callback: &'self fn(&ast::view_item) -> bool, } -impl SimpleVisitor for EachViewItemData { - fn visit_mod(&mut self, _: &_mod, _: Span, _: NodeId) { - // XXX: Default method. - } - fn visit_view_item(&mut self, view_item: &view_item) { +impl<'self> Visitor<()> for EachViewItemData<'self> { + fn visit_view_item(&mut self, view_item: &ast::view_item, _: ()) { let _ = (self.callback)(view_item); } - fn visit_foreign_item(&mut self, _: @foreign_item) { - // XXX: Default method. - } - fn visit_item(&mut self, _: @item) { - // XXX: Default method. - } - fn visit_local(&mut self, _: @Local) { - // XXX: Default method. - } - fn visit_block(&mut self, _: &Block) { - // XXX: Default method. - } - fn visit_stmt(&mut self, _: @Stmt) { - // XXX: Default method. - } - fn visit_arm(&mut self, _: &Arm) { - // XXX: Default method. - } - fn visit_pat(&mut self, _: @Pat) { - // XXX: Default method. - } - fn visit_decl(&mut self, _: @Decl) { - // XXX: Default method. - } - fn visit_expr(&mut self, _: @Expr) { - // XXX: Default method. - } - fn visit_expr_post(&mut self, _: @Expr) { - // XXX: Default method. - } - fn visit_ty(&mut self, _: &Ty) { - // XXX: Default method. - } - fn visit_generics(&mut self, _: &Generics) { - // XXX: Default method. - } - fn visit_fn(&mut self, - _: &visit::fn_kind, - _: &fn_decl, - _: &Block, - _: Span, - _: NodeId) { - // XXX: Default method. - } - fn visit_ty_method(&mut self, _: &TypeMethod) { - // XXX: Default method. - } - fn visit_trait_method(&mut self, _: &trait_method) { - // XXX: Default method. - } - fn visit_struct_def(&mut self, - _: @struct_def, - _: Ident, - _: &Generics, - _: NodeId) { - // XXX: Default method. - } - fn visit_struct_field(&mut self, _: @struct_field) { - // XXX: Default method. - } - fn visit_struct_method(&mut self, _: @method) { - // XXX: Default method. - } } impl EachViewItem for ast::Crate { - fn each_view_item(&self, f: @fn(&ast::view_item) -> bool) -> bool { - let data = @mut EachViewItemData { + fn each_view_item(&self, f: &fn(&ast::view_item) -> bool) -> bool { + let mut visit = EachViewItemData { callback: f, }; - let visitor = @mut SimpleVisitorVisitor { - simple_visitor: data as @mut SimpleVisitor, - }; - visit::walk_crate(visitor, self, ()); + visit::walk_crate(&mut visit, self, ()); true } }