auto merge of #9146 : catamorphism/rust/rustpkg-sub-package-ids, r=brson,catamorphism

r? @brson Package IDs can now refer to a subdirectory of a particular source
tree, and not just a top-level directory with a src/ directory as its
parent.

For example, referring to the package ID a/b/c/d , in workspace W,
if W/src/a is a package, will build the sources for the package in
a/b/c/d (and not other crates in W/src/a).

Closes #6408

src/librustpkg/package_id.rs

@@ -99,7 +99,35 @@ impl PkgId {
     /// True if the ID has multiple components
     pub fn is_complex(&self) -> bool {
         self.short_name != self.path.to_str()
-     }
+    }
+
+    pub fn prefixes_iter(&self) -> Prefixes {
+        Prefixes {
+            components: self.path.components().to_owned(),
+            remaining: ~[]
+        }
+    }
+
+}
+
+struct Prefixes {
+    priv components: ~[~str],
+    priv remaining: ~[~str]
+}
+
+impl Iterator<(Path, Path)> for Prefixes {
+    #[inline]
+    fn next(&mut self) -> Option<(Path, Path)> {
+        if self.components.len() <= 1 {
+            None
+        }
+        else {
+            let last = self.components.pop();
+            self.remaining.push(last);
+            // converting to str and then back is a little unfortunate
+            Some((Path(self.components.to_str()), Path(self.remaining.to_str())))
+        }
+    }
 }
 
 impl ToStr for PkgId {
@@ -119,3 +147,4 @@ pub fn hash(data: ~str) -> ~str {
     write(hasher, data);
     hasher.result_str()
 }
+

src/librustpkg/package_source.rs

@@ -77,6 +77,33 @@ impl PkgSrc {
         let dir: Path = match path {
             Some(d) => (*d).clone(),
             None => {
+                // See if any of the prefixes of this package ID form a valid package ID
+                // That is, is this a package ID that points into the middle of a workspace?
+                for (prefix, suffix) in id.prefixes_iter() {
+                    let package_id = PkgId::new(prefix.to_str());
+                    let path = workspace.push("src").push_rel(&package_id.path);
+                    debug!("in loop: checking if %s is a directory", path.to_str());
+                    if os::path_is_dir(&path) {
+                        let ps = PkgSrc::new(workspace.clone(),
+                                             use_rust_path_hack,
+                                             PkgId::new(prefix.to_str()));
+                        debug!("pkgsrc: Returning [%s|%s|%s]", workspace.to_str(),
+                               ps.start_dir.push_rel(&suffix).to_str(), ps.id.to_str());
+
+                        return PkgSrc {
+                            workspace: workspace,
+                            start_dir: ps.start_dir.push_rel(&suffix),
+                            id: ps.id,
+                            libs: ~[],
+                            mains: ~[],
+                            tests: ~[],
+                            benchs: ~[]
+                        }
+
+                    };
+                }
+
+                // Ok, no prefixes work, so try fetching from git
                 let mut ok_d = None;
                 for w in to_try.iter() {
                     debug!("Calling fetch_git on %s", w.to_str());
@@ -93,16 +120,17 @@ impl PkgSrc {
                         if use_rust_path_hack {
                             match find_dir_using_rust_path_hack(&id) {
                                 Some(d) => d,
-                                None => cond.raise((id.clone(),
-                                    ~"supplied path for package dir does not \
-                                     exist, and couldn't interpret it as a URL fragment"))
+                                None => {
+                                    cond.raise((id.clone(),
+                                        ~"supplied path for package dir does not \
+                                        exist, and couldn't interpret it as a URL fragment"))
+                                }
                             }
                         }
                         else {
                             cond.raise((id.clone(),
-                                        ~"supplied path for package dir does not \
-                                        exist, and couldn't interpret it as a URL fragment"))
-
+                                ~"supplied path for package dir does not \
+                                exist, and couldn't interpret it as a URL fragment"))
                         }
                     }
                 }
@@ -115,6 +143,9 @@ impl PkgSrc {
                                         non-directory"));
         }
 
+        debug!("pkgsrc: Returning {%s|%s|%s}", workspace.to_str(),
+               dir.to_str(), id.to_str());
+
         PkgSrc {
             workspace: workspace,
             start_dir: dir,

src/librustpkg/path_util.rs

@@ -49,8 +49,6 @@ pub fn make_dir_rwx(p: &Path) -> bool { os::make_dir(p, U_RWX) }
 /// True if there's a directory in <workspace> with
 /// pkgid's short name
 pub fn workspace_contains_package_id(pkgid: &PkgId, workspace: &Path) -> bool {
-    debug!("Checking in src dir of %s for %s",
-           workspace.to_str(), pkgid.to_str());
     workspace_contains_package_id_(pkgid, workspace, |p| { p.push("src") }).is_some()
 }
 
@@ -141,9 +139,17 @@ pub fn built_library_in_workspace(pkgid: &PkgId, workspace: &Path) -> Option<Pat
 }
 
 /// Does the actual searching stuff
-pub fn installed_library_in_workspace(short_name: &str, workspace: &Path) -> Option<Path> {
+pub fn installed_library_in_workspace(pkg_path: &Path, workspace: &Path) -> Option<Path> {
     // This could break once we're handling multiple versions better -- I should add a test for it
-    library_in_workspace(&Path(short_name), short_name, Install, workspace, "lib", &NoVersion)
+    match pkg_path.filename() {
+        None => None,
+        Some(short_name) => library_in_workspace(pkg_path,
+                                                 short_name,
+                                                 Install,
+                                                 workspace,
+                                                 "lib",
+                                                 &NoVersion)
+    }
 }
 
 /// `workspace` is used to figure out the directory to search.

src/librustpkg/search.rs

@@ -15,10 +15,11 @@ use version::Version;
 /// return Some(p) (returns the first one of there are multiple matches.) Return
 /// None if there's no such path.
 /// FIXME #8711: This ignores the desired version.
-pub fn find_installed_library_in_rust_path(short_name: &str, _version: &Version) -> Option<Path> {
+pub fn find_installed_library_in_rust_path(pkg_path: &Path, _version: &Version) -> Option<Path> {
     let rp = rust_path();
+    debug!("find_installed_library_in_rust_path: looking for path %s", pkg_path.to_str());
     for p in rp.iter() {
-        match installed_library_in_workspace(short_name, p) {
+        match installed_library_in_workspace(pkg_path, p) {
             Some(path) => return Some(path),
             None => ()
         }

src/librustpkg/tests.rs

@@ -328,13 +328,13 @@ fn create_local_package_with_custom_build_hook(pkgid: &PkgId,
 
 }
 
-fn assert_lib_exists(repo: &Path, short_name: &str, v: Version) {
-    assert!(lib_exists(repo, short_name, v));
+fn assert_lib_exists(repo: &Path, pkg_path: &Path, v: Version) {
+    assert!(lib_exists(repo, pkg_path, v));
 }
 
-fn lib_exists(repo: &Path, short_name: &str, _v: Version) -> bool { // ??? version?
-    debug!("assert_lib_exists: repo = %s, short_name = %s", repo.to_str(), short_name);
-    let lib = installed_library_in_workspace(short_name, repo);
+fn lib_exists(repo: &Path, pkg_path: &Path, _v: Version) -> bool { // ??? version?
+    debug!("assert_lib_exists: repo = %s, pkg_path = %s", repo.to_str(), pkg_path.to_str());
+    let lib = installed_library_in_workspace(pkg_path, repo);
     debug!("assert_lib_exists: checking whether %? exists", lib);
     lib.is_some() && {
         let libname = lib.get_ref();
@@ -507,7 +507,7 @@ fn test_install_valid() {
     assert!(os::path_exists(&exec));
     assert!(is_rwx(&exec));
 
-    let lib = installed_library_in_workspace(temp_pkg_id.short_name, &temp_workspace);
+    let lib = installed_library_in_workspace(&temp_pkg_id.path, &temp_workspace);
     debug!("lib = %?", lib);
     assert!(lib.map_default(false, |l| os::path_exists(l)));
     assert!(lib.map_default(false, |l| is_rwx(l)));
@@ -571,7 +571,7 @@ fn test_install_git() {
     let _built_lib =
         built_library_in_workspace(&temp_pkg_id,
                                    &ws).expect("test_install_git: built lib should exist");
-    assert_lib_exists(&ws, temp_pkg_id.short_name, temp_pkg_id.version.clone());
+    assert_lib_exists(&ws, &temp_pkg_id.path, temp_pkg_id.version.clone());
     let built_test = built_test_in_workspace(&temp_pkg_id,
                          &ws).expect("test_install_git: built test should exist");
     assert!(os::path_exists(&built_test));
@@ -685,7 +685,7 @@ fn test_package_request_version() {
 
     command_line_test([~"install", fmt!("%s#0.3", local_path)], &repo);
 
-    assert!(match installed_library_in_workspace("test_pkg_version", &repo.push(".rust")) {
+    assert!(match installed_library_in_workspace(&Path("test_pkg_version"), &repo.push(".rust")) {
         Some(p) => {
             debug!("installed: %s", p.to_str());
             p.to_str().ends_with(fmt!("0.3%s", os::consts::DLL_SUFFIX))
@@ -731,7 +731,7 @@ fn rustpkg_library_target() {
 
     add_git_tag(&package_dir, ~"1.0");
     command_line_test([~"install", ~"foo"], &foo_repo);
-    assert_lib_exists(&foo_repo.push(".rust"), "foo", ExactRevision(~"1.0"));
+    assert_lib_exists(&foo_repo.push(".rust"), &Path("foo"), ExactRevision(~"1.0"));
 }
 
 #[test]
@@ -754,7 +754,7 @@ fn package_script_with_default_build() {
         fail!("Couldn't copy file");
     }
     command_line_test([~"install", ~"fancy-lib"], &dir);
-    assert_lib_exists(&dir, "fancy-lib", NoVersion);
+    assert_lib_exists(&dir, &Path("fancy-lib"), NoVersion);
     assert!(os::path_exists(&dir.push("build").push("fancy-lib").push("generated.rs")));
 }
 
@@ -783,7 +783,7 @@ fn rustpkg_install_no_arg() {
               "fn main() { let _x = (); }");
     debug!("install_no_arg: dir = %s", package_dir.to_str());
     command_line_test([~"install"], &package_dir);
-    assert_lib_exists(&tmp, "foo", NoVersion);
+    assert_lib_exists(&tmp, &Path("foo"), NoVersion);
 }
 
 #[test]
@@ -1172,11 +1172,11 @@ fn rust_path_hack_test(hack_flag: bool) {
        dest_workspace.to_str(), workspace.push_many(["src", "foo-0.1"]).to_str());
    command_line_test_with_env(~[~"install"] + if hack_flag { ~[~"--rust-path-hack"] } else { ~[] } +
                                ~[~"foo"], &dest_workspace, rust_path);
-   assert_lib_exists(&dest_workspace, "foo", NoVersion);
+   assert_lib_exists(&dest_workspace, &Path("foo"), NoVersion);
    assert_executable_exists(&dest_workspace, "foo");
    assert_built_library_exists(&dest_workspace, "foo");
    assert_built_executable_exists(&dest_workspace, "foo");
-   assert!(!lib_exists(&workspace, "foo", NoVersion));
+   assert!(!lib_exists(&workspace, &Path("foo"), NoVersion));
    assert!(!executable_exists(&workspace, "foo"));
    assert!(!built_library_exists(&workspace, "foo"));
    assert!(!built_executable_exists(&workspace, "foo"));
@@ -1212,9 +1212,9 @@ fn rust_path_hack_cwd() {
    debug!("declare -x RUST_PATH=%s", dest_workspace.to_str());
    command_line_test_with_env([~"install", ~"--rust-path-hack", ~"foo"], &cwd, rust_path);
    debug!("Checking that foo exists in %s", dest_workspace.to_str());
-   assert_lib_exists(&dest_workspace, "foo", NoVersion);
+   assert_lib_exists(&dest_workspace, &Path("foo"), NoVersion);
    assert_built_library_exists(&dest_workspace, "foo");
-   assert!(!lib_exists(&cwd, "foo", NoVersion));
+   assert!(!lib_exists(&cwd, &Path("foo"), NoVersion));
    assert!(!built_library_exists(&cwd, "foo"));
 }
 
@@ -1232,9 +1232,9 @@ fn rust_path_hack_multi_path() {
    debug!("declare -x RUST_PATH=%s", dest_workspace.to_str());
    command_line_test_with_env([~"install", ~"--rust-path-hack", name.clone()], &subdir, rust_path);
    debug!("Checking that %s exists in %s", name, dest_workspace.to_str());
-   assert_lib_exists(&dest_workspace, "quux", NoVersion);
+   assert_lib_exists(&dest_workspace, &Path("quux"), NoVersion);
    assert_built_library_exists(&dest_workspace, name);
-   assert!(!lib_exists(&subdir, "quux", NoVersion));
+   assert!(!lib_exists(&subdir, &Path("quux"), NoVersion));
    assert!(!built_library_exists(&subdir, name));
 }
 
@@ -1251,9 +1251,9 @@ fn rust_path_hack_install_no_arg() {
    debug!("declare -x RUST_PATH=%s", dest_workspace.to_str());
    command_line_test_with_env([~"install", ~"--rust-path-hack"], &source_dir, rust_path);
    debug!("Checking that foo exists in %s", dest_workspace.to_str());
-   assert_lib_exists(&dest_workspace, "foo", NoVersion);
+   assert_lib_exists(&dest_workspace, &Path("foo"), NoVersion);
    assert_built_library_exists(&dest_workspace, "foo");
-   assert!(!lib_exists(&source_dir, "foo", NoVersion));
+   assert!(!lib_exists(&source_dir, &Path("foo"), NoVersion));
    assert!(!built_library_exists(&cwd, "foo"));
 }
 
@@ -1378,7 +1378,7 @@ fn notrans_flag_fail() {
         // we can't tell
         assert!(!built_executable_exists(&workspace, "foo"));
         assert!(!object_file_exists(&workspace, "foo"));
-        assert!(!lib_exists(&workspace, "foo", NoVersion));
+        assert!(!lib_exists(&workspace, &Path("foo"), NoVersion));
     }
 }
 
@@ -1550,6 +1550,37 @@ fn test_optimized_build() {
     assert!(built_executable_exists(&workspace, "foo"));
 }
 
+fn pkgid_pointing_to_subdir() {
+    // The actual repo is mockgithub.com/mozilla/some_repo
+    // rustpkg should recognize that and treat the part after some_repo/ as a subdir
+    let workspace = mkdtemp(&os::tmpdir(), "parent_repo").expect("Couldn't create temp dir");
+    assert!(os::mkdir_recursive(&workspace.push_many([~"src", ~"mockgithub.com",
+                                                     ~"mozilla", ~"some_repo"]), U_RWX));
+
+    let foo_dir = workspace.push_many([~"src", ~"mockgithub.com", ~"mozilla", ~"some_repo",
+                                       ~"extras", ~"foo"]);
+    let bar_dir = workspace.push_many([~"src", ~"mockgithub.com", ~"mozilla", ~"some_repo",
+                                       ~"extras", ~"bar"]);
+    assert!(os::mkdir_recursive(&foo_dir, U_RWX));
+    assert!(os::mkdir_recursive(&bar_dir, U_RWX));
+    writeFile(&foo_dir.push("lib.rs"), "pub fn f() {}");
+    writeFile(&bar_dir.push("lib.rs"), "pub fn g() {}");
+
+    debug!("Creating a file in %s", workspace.to_str());
+    let testpkg_dir = workspace.push_many([~"src", ~"testpkg-0.1"]);
+    assert!(os::mkdir_recursive(&testpkg_dir, U_RWX));
+
+    writeFile(&testpkg_dir.push("main.rs"),
+              "extern mod foo = \"mockgithub.com/mozilla/some_repo/extras/foo\";\n
+               extern mod bar = \"mockgithub.com/mozilla/some_repo/extras/bar\";\n
+               use foo::f; use bar::g; \n
+               fn main() { f(); g(); }");
+
+    debug!("RUST_PATH=%s", workspace.to_str());
+    command_line_test([~"install", ~"testpkg"], &workspace);
+    assert_executable_exists(&workspace, "testpkg");
+}
+
 /// Returns true if p exists and is executable
 fn is_executable(p: &Path) -> bool {
     use std::libc::consts::os::posix88::{S_IXUSR};

src/librustpkg/util.rs

@@ -378,7 +378,8 @@ pub fn find_and_install_dependencies(ctxt: &BuildContext,
                     Some(p) => p,
                     None => sess.str_of(lib_ident)
                 };
-                match installed_library_in_workspace(lib_name, &ctxt.sysroot()) {
+                debug!("Finding and installing... %s", lib_name);
+                match installed_library_in_workspace(&Path(lib_name), &ctxt.sysroot()) {
                     Some(ref installed_path) => {
                         debug!("It exists: %s", installed_path.to_str());
                         // Say that [path for c] has a discovered dependency on
@@ -420,8 +421,9 @@ pub fn find_and_install_dependencies(ctxt: &BuildContext,
                             }
                         }
                         // Also, add an additional search path
+                        debug!("Adding additional search path: %s", lib_name);
                         let installed_library =
-                            installed_library_in_workspace(lib_name, workspace)
+                            installed_library_in_workspace(&Path(lib_name), workspace)
                                 .expect( fmt!("rustpkg failed to install dependency %s",
                                               lib_name));
                         let install_dir = installed_library.pop();

src/librustpkg/workspace.rs

@@ -38,6 +38,8 @@ pub fn each_pkg_parent_workspace(cx: &Context, pkgid: &PkgId, action: &fn(&Path)
     return true;
 }
 
+/// Given a package ID, return a vector of all of the workspaces in
+/// the RUST_PATH that contain it
 pub fn pkg_parent_workspaces(cx: &Context, pkgid: &PkgId) -> ~[Path] {
     let rs: ~[Path] = rust_path().move_iter()
         .filter(|ws| workspace_contains_package_id(pkgid, ws))