Auto merge of #22235 - michaelwoerister:cross-crate-spans, r=michaelwoerister

This allows to create proper debuginfo line information for items inlined from other crates (e.g. instantiations of generics). Only the codemap's 'metadata' is stored in a crate's metadata. That is, just filename, positions of line-beginnings, etc. but not the actual source code itself.

Crate metadata size is increased by this change because spans in the encoded ASTs take up space now:
```
                BEFORE    AFTER
libcore         36 MiB    39.6 MiB    +10%
libsyntax       51.1 MiB  60.5 MiB    +18.4%
libcollections  11.2 MiB  12.8 MiB    +14.3%
```
This only affects binaries containing metadata (rlibs and dylibs), executables should not be affected in size. 

Fixes #19228 and probably #22226.

src/librustc/metadata/common.rs

@@ -252,3 +252,6 @@ pub const tag_macro_def: uint = 0x9e;
 pub const tag_macro_def_body: uint = 0x9f;
 
 pub const tag_paren_sugar: uint = 0xa0;
+
+pub const tag_codemap: uint = 0xa1;
+pub const tag_codemap_filemap: uint = 0xa2;

src/librustc/metadata/creader.rs

@@ -26,7 +26,7 @@ use syntax::ast;
 use syntax::abi;
 use syntax::attr;
 use syntax::attr::AttrMetaMethods;
-use syntax::codemap::{Span, mk_sp};
+use syntax::codemap::{self, Span, mk_sp, Pos};
 use syntax::parse;
 use syntax::parse::token::InternedString;
 use syntax::parse::token;
@@ -373,15 +373,17 @@ impl<'a> CrateReader<'a> {
         // Maintain a reference to the top most crate.
         let root = if root.is_some() { root } else { &crate_paths };
 
-        let cnum_map = self.resolve_crate_deps(root, lib.metadata.as_slice(), span);
-
         let loader::Library { dylib, rlib, metadata } = lib;
 
+        let cnum_map = self.resolve_crate_deps(root, metadata.as_slice(), span);
+        let codemap_import_info = import_codemap(self.sess.codemap(), &metadata);
+
         let cmeta = Rc::new( cstore::crate_metadata {
             name: name.to_string(),
             data: metadata,
             cnum_map: cnum_map,
             cnum: cnum,
+            codemap_import_info: codemap_import_info,
             span: span,
         });
 
@@ -586,3 +588,131 @@ impl<'a> CrateReader<'a> {
         }
     }
 }
+
+/// Imports the codemap from an external crate into the codemap of the crate
+/// currently being compiled (the "local crate").
+///
+/// The import algorithm works analogous to how AST items are inlined from an
+/// external crate's metadata:
+/// For every FileMap in the external codemap an 'inline' copy is created in the
+/// local codemap. The correspondence relation between external and local
+/// FileMaps is recorded in the `ImportedFileMap` objects returned from this
+/// function. When an item from an external crate is later inlined into this
+/// crate, this correspondence information is used to translate the span
+/// information of the inlined item so that it refers the correct positions in
+/// the local codemap (see `astencode::DecodeContext::tr_span()`).
+///
+/// The import algorithm in the function below will reuse FileMaps already
+/// existing in the local codemap. For example, even if the FileMap of some
+/// source file of libstd gets imported many times, there will only ever be
+/// one FileMap object for the corresponding file in the local codemap.
+///
+/// Note that imported FileMaps do not actually contain the source code of the
+/// file they represent, just information about length, line breaks, and
+/// multibyte characters. This information is enough to generate valid debuginfo
+/// for items inlined from other crates.
+fn import_codemap(local_codemap: &codemap::CodeMap,
+                  metadata: &MetadataBlob)
+                  -> Vec<cstore::ImportedFileMap> {
+    let external_codemap = decoder::get_imported_filemaps(metadata.as_slice());
+
+    let imported_filemaps = external_codemap.into_iter().map(|filemap_to_import| {
+        // Try to find an existing FileMap that can be reused for the filemap to
+        // be imported. A FileMap is reusable if it is exactly the same, just
+        // positioned at a different offset within the codemap.
+        let reusable_filemap = {
+            local_codemap.files
+                         .borrow()
+                         .iter()
+                         .find(|fm| are_equal_modulo_startpos(&fm, &filemap_to_import))
+                         .map(|rc| rc.clone())
+        };
+
+        match reusable_filemap {
+            Some(fm) => {
+                cstore::ImportedFileMap {
+                    original_start_pos: filemap_to_import.start_pos,
+                    original_end_pos: filemap_to_import.end_pos,
+                    translated_filemap: fm
+                }
+            }
+            None => {
+                // We can't reuse an existing FileMap, so allocate a new one
+                // containing the information we need.
+                let codemap::FileMap {
+                    name,
+                    start_pos,
+                    end_pos,
+                    lines,
+                    multibyte_chars,
+                    ..
+                } = filemap_to_import;
+
+                let source_length = (end_pos - start_pos).to_usize();
+
+                // Translate line-start positions and multibyte character
+                // position into frame of reference local to file.
+                // `CodeMap::new_imported_filemap()` will then translate those
+                // coordinates to their new global frame of reference when the
+                // offset of the FileMap is known.
+                let lines = lines.into_inner().map_in_place(|pos| pos - start_pos);
+                let multibyte_chars = multibyte_chars
+                    .into_inner()
+                    .map_in_place(|mbc|
+                        codemap::MultiByteChar {
+                            pos: mbc.pos + start_pos,
+                            bytes: mbc.bytes
+                        });
+
+                let local_version = local_codemap.new_imported_filemap(name,
+                                                                       source_length,
+                                                                       lines,
+                                                                       multibyte_chars);
+                cstore::ImportedFileMap {
+                    original_start_pos: start_pos,
+                    original_end_pos: end_pos,
+                    translated_filemap: local_version
+                }
+            }
+        }
+    }).collect();
+
+    return imported_filemaps;
+
+    fn are_equal_modulo_startpos(fm1: &codemap::FileMap,
+                                 fm2: &codemap::FileMap)
+                                 -> bool {
+        if fm1.name != fm2.name {
+            return false;
+        }
+
+        let lines1 = fm1.lines.borrow();
+        let lines2 = fm2.lines.borrow();
+
+        if lines1.len() != lines2.len() {
+            return false;
+        }
+
+        for (&line1, &line2) in lines1.iter().zip(lines2.iter()) {
+            if (line1 - fm1.start_pos) != (line2 - fm2.start_pos) {
+                return false;
+            }
+        }
+
+        let multibytes1 = fm1.multibyte_chars.borrow();
+        let multibytes2 = fm2.multibyte_chars.borrow();
+
+        if multibytes1.len() != multibytes2.len() {
+            return false;
+        }
+
+        for (mb1, mb2) in multibytes1.iter().zip(multibytes2.iter()) {
+            if (mb1.bytes != mb2.bytes) ||
+               ((mb1.pos - fm1.start_pos) != (mb2.pos - fm2.start_pos)) {
+                return false;
+            }
+        }
+
+        true
+    }
+}

src/librustc/metadata/cstore.rs

@@ -27,7 +27,7 @@ use std::cell::RefCell;
 use std::rc::Rc;
 use flate::Bytes;
 use syntax::ast;
-use syntax::codemap::Span;
+use syntax::codemap;
 use syntax::parse::token::IdentInterner;
 
 // A map from external crate numbers (as decoded from some crate file) to
@@ -41,12 +41,24 @@ pub enum MetadataBlob {
     MetadataArchive(loader::ArchiveMetadata),
 }
 
+/// Holds information about a codemap::FileMap imported from another crate.
+/// See creader::import_codemap() for more information.
+pub struct ImportedFileMap {
+    /// This FileMap's byte-offset within the codemap of its original crate
+    pub original_start_pos: codemap::BytePos,
+    /// The end of this FileMap within the codemap of its original crate
+    pub original_end_pos: codemap::BytePos,
+    /// The imported FileMap's representation within the local codemap
+    pub translated_filemap: Rc<codemap::FileMap>
+}
+
 pub struct crate_metadata {
     pub name: String,
     pub data: MetadataBlob,
     pub cnum_map: cnum_map,
     pub cnum: ast::CrateNum,
-    pub span: Span,
+    pub codemap_import_info: Vec<ImportedFileMap>,
+    pub span: codemap::Span,
 }
 
 #[derive(Copy, Debug, PartialEq, Clone)]

src/librustc/metadata/decoder.rs

@@ -1561,7 +1561,6 @@ pub fn is_associated_type(cdata: Cmd, id: ast::NodeId) -> bool {
     }
 }
 
-
 pub fn is_default_trait<'tcx>(cdata: Cmd, id: ast::NodeId) -> bool {
     let item_doc = lookup_item(id, cdata.data());
     match item_family(item_doc) {
@@ -1569,3 +1568,19 @@ pub fn is_default_trait<'tcx>(cdata: Cmd, id: ast::NodeId) -> bool {
         _ => false
     }
 }
+
+pub fn get_imported_filemaps(metadata: &[u8]) -> Vec<codemap::FileMap> {
+    let crate_doc = rbml::Doc::new(metadata);
+    let cm_doc = reader::get_doc(crate_doc, tag_codemap);
+
+    let mut filemaps = vec![];
+
+    reader::tagged_docs(cm_doc, tag_codemap_filemap, |filemap_doc| {
+        let mut decoder = reader::Decoder::new(filemap_doc);
+        let filemap: codemap::FileMap = Decodable::decode(&mut decoder).unwrap();
+        filemaps.push(filemap);
+        true
+    });
+
+    return filemaps;
+}

src/librustc/metadata/encoder.rs

@@ -1751,6 +1751,28 @@ fn encode_plugin_registrar_fn(ecx: &EncodeContext, rbml_w: &mut Encoder) {
     }
 }
 
+fn encode_codemap(ecx: &EncodeContext, rbml_w: &mut Encoder) {
+    rbml_w.start_tag(tag_codemap);
+    let codemap = ecx.tcx.sess.codemap();
+
+    for filemap in &codemap.files.borrow()[..] {
+
+        if filemap.lines.borrow().len() == 0 || filemap.is_imported() {
+            // No need to export empty filemaps, as they can't contain spans
+            // that need translation.
+            // Also no need to re-export imported filemaps, as any downstream
+            // crate will import them from their original source.
+            continue;
+        }
+
+        rbml_w.start_tag(tag_codemap_filemap);
+        filemap.encode(rbml_w);
+        rbml_w.end_tag();
+    }
+
+    rbml_w.end_tag();
+}
+
 /// Serialize the text of the exported macros
 fn encode_macro_defs(rbml_w: &mut Encoder,
                      krate: &ast::Crate) {
@@ -1968,6 +1990,7 @@ fn encode_metadata_inner(wr: &mut SeekableMemWriter,
         lang_item_bytes: u64,
         native_lib_bytes: u64,
         plugin_registrar_fn_bytes: u64,
+        codemap_bytes: u64,
         macro_defs_bytes: u64,
         impl_bytes: u64,
         misc_bytes: u64,
@@ -1982,6 +2005,7 @@ fn encode_metadata_inner(wr: &mut SeekableMemWriter,
         lang_item_bytes: 0,
         native_lib_bytes: 0,
         plugin_registrar_fn_bytes: 0,
+        codemap_bytes: 0,
         macro_defs_bytes: 0,
         impl_bytes: 0,
         misc_bytes: 0,
@@ -2047,6 +2071,11 @@ fn encode_metadata_inner(wr: &mut SeekableMemWriter,
     encode_plugin_registrar_fn(&ecx, &mut rbml_w);
     stats.plugin_registrar_fn_bytes = rbml_w.writer.tell().unwrap() - i;
 
+    // Encode codemap
+    i = rbml_w.writer.tell().unwrap();
+    encode_codemap(&ecx, &mut rbml_w);
+    stats.codemap_bytes = rbml_w.writer.tell().unwrap() - i;
+
     // Encode macro definitions
     i = rbml_w.writer.tell().unwrap();
     encode_macro_defs(&mut rbml_w, krate);
@@ -2091,6 +2120,7 @@ fn encode_metadata_inner(wr: &mut SeekableMemWriter,
         println!("       lang item bytes: {}", stats.lang_item_bytes);
         println!("          native bytes: {}", stats.native_lib_bytes);
         println!("plugin registrar bytes: {}", stats.plugin_registrar_fn_bytes);
+        println!("         codemap bytes: {}", stats.codemap_bytes);
         println!("       macro def bytes: {}", stats.macro_defs_bytes);
         println!("            impl bytes: {}", stats.impl_bytes);
         println!("            misc bytes: {}", stats.misc_bytes);

src/librustc/middle/astencode.rs

@@ -42,6 +42,7 @@ use syntax;
 use std::old_io::Seek;
 use std::num::FromPrimitive;
 use std::rc::Rc;
+use std::cell::Cell;
 
 use rbml::reader;
 use rbml::writer::Encoder;
@@ -58,7 +59,9 @@ struct DecodeContext<'a, 'b, 'tcx: 'a> {
     tcx: &'a ty::ctxt<'tcx>,
     cdata: &'b cstore::crate_metadata,
     from_id_range: ast_util::IdRange,
-    to_id_range: ast_util::IdRange
+    to_id_range: ast_util::IdRange,
+    // Cache the last used filemap for translating spans as an optimization.
+    last_filemap_index: Cell<usize>,
 }
 
 trait tr {
@@ -120,6 +123,8 @@ impl<'a, 'b, 'c, 'tcx> ast_map::FoldOps for &'a DecodeContext<'b, 'c, 'tcx> {
     }
 }
 
+/// Decodes an item from its AST in the cdata's metadata and adds it to the
+/// ast-map.
 pub fn decode_inlined_item<'tcx>(cdata: &cstore::crate_metadata,
                                  tcx: &ty::ctxt<'tcx>,
                                  path: Vec<ast_map::PathElem>,
@@ -143,7 +148,8 @@ pub fn decode_inlined_item<'tcx>(cdata: &cstore::crate_metadata,
             cdata: cdata,
             tcx: tcx,
             from_id_range: from_id_range,
-            to_id_range: to_id_range
+            to_id_range: to_id_range,
+            last_filemap_index: Cell::new(0)
         };
         let raw_ii = decode_ast(ast_doc);
         let ii = ast_map::map_decoded_item(&dcx.tcx.map, path, raw_ii, dcx);
@@ -234,8 +240,47 @@ impl<'a, 'b, 'tcx> DecodeContext<'a, 'b, 'tcx> {
         assert_eq!(did.krate, ast::LOCAL_CRATE);
         ast::DefId { krate: ast::LOCAL_CRATE, node: self.tr_id(did.node) }
     }
-    pub fn tr_span(&self, _span: Span) -> Span {
-        codemap::DUMMY_SP // FIXME (#1972): handle span properly
+
+    /// Translates a `Span` from an extern crate to the corresponding `Span`
+    /// within the local crate's codemap. `creader::import_codemap()` will
+    /// already have allocated any additionally needed FileMaps in the local
+    /// codemap as a side-effect of creating the crate_metadata's
+    /// `codemap_import_info`.
+    pub fn tr_span(&self, span: Span) -> Span {
+        let imported_filemaps = &self.cdata.codemap_import_info[..];
+
+        let filemap_index = {
+            // Optimize for the case that most spans within a translated item
+            // originate from the same filemap.
+            let last_filemap_index = self.last_filemap_index.get();
+
+            if span.lo >= imported_filemaps[last_filemap_index].original_start_pos &&
+               span.hi <= imported_filemaps[last_filemap_index].original_end_pos {
+                last_filemap_index
+            } else {
+                let mut a = 0;
+                let mut b = imported_filemaps.len();
+
+                while b - a > 1 {
+                    let m = (a + b) / 2;
+                    if imported_filemaps[m].original_start_pos > span.lo {
+                        b = m;
+                    } else {
+                        a = m;
+                    }
+                }
+
+                self.last_filemap_index.set(a);
+                a
+            }
+        };
+
+        let lo = (span.lo - imported_filemaps[filemap_index].original_start_pos) +
+                  imported_filemaps[filemap_index].translated_filemap.start_pos;
+        let hi = (span.hi - imported_filemaps[filemap_index].original_start_pos) +
+                  imported_filemaps[filemap_index].translated_filemap.start_pos;
+
+        codemap::mk_sp(lo, hi)
     }
 }
 

src/librustc_driver/pretty.rs

@@ -542,7 +542,11 @@ pub fn pretty_print_input(sess: Session,
 
     let src_name = driver::source_name(input);
     let src = sess.codemap().get_filemap(&src_name[..])
-                            .src.as_bytes().to_vec();
+                            .src
+                            .as_ref()
+                            .unwrap()
+                            .as_bytes()
+                            .to_vec();
     let mut rdr = MemReader::new(src);
 
     let out = match ofile {

src/libsyntax/codemap.rs

@@ -29,6 +29,11 @@ use std::rc::Rc;
 use libc::c_uint;
 use serialize::{Encodable, Decodable, Encoder, Decoder};
 
+
+// _____________________________________________________________________________
+// Pos, BytePos, CharPos
+//
+
 pub trait Pos {
     fn from_usize(n: usize) -> Self;
     fn to_usize(&self) -> usize;
@@ -69,6 +74,18 @@ impl Sub for BytePos {
     }
 }
 
+impl Encodable for BytePos {
+    fn encode<S: Encoder>(&self, s: &mut S) -> Result<(), S::Error> {
+        s.emit_u32(self.0)
+    }
+}
+
+impl Decodable for BytePos {
+    fn decode<D: Decoder>(d: &mut D) -> Result<BytePos, D::Error> {
+        Ok(BytePos(try!{ d.read_u32() }))
+    }
+}
+
 impl Pos for CharPos {
     fn from_usize(n: usize) -> CharPos { CharPos(n) }
     fn to_usize(&self) -> usize { let CharPos(n) = *self; n }
@@ -90,6 +107,10 @@ impl Sub for CharPos {
     }
 }
 
+// _____________________________________________________________________________
+// Span, Spanned
+//
+
 /// Spans represent a region of code, used for error reporting. Positions in spans
 /// are *absolute* positions from the beginning of the codemap, not positions
 /// relative to FileMaps. Methods on the CodeMap can be used to relate spans back
@@ -126,15 +147,20 @@ impl PartialEq for Span {
 impl Eq for Span {}
 
 impl Encodable for Span {
-    /* Note #1972 -- spans are encoded but not decoded */
     fn encode<S: Encoder>(&self, s: &mut S) -> Result<(), S::Error> {
-        s.emit_nil()
+        // Encode spans as a single u64 in order to cut down on tagging overhead
+        // added by the RBML metadata encoding. The should be solved differently
+        // altogether some time (FIXME #21482)
+        s.emit_u64( (self.lo.0 as u64) | ((self.hi.0 as u64) << 32) )
     }
 }
 
 impl Decodable for Span {
-    fn decode<D: Decoder>(_d: &mut D) -> Result<Span, D::Error> {
-        Ok(DUMMY_SP)
+    fn decode<D: Decoder>(d: &mut D) -> Result<Span, D::Error> {
+        let lo_hi: u64 = try! { d.read_u64() };
+        let lo = BytePos(lo_hi as u32);
+        let hi = BytePos((lo_hi >> 32) as u32);
+        Ok(mk_sp(lo, hi))
     }
 }
 
@@ -168,6 +194,10 @@ pub fn original_sp(cm: &CodeMap, sp: Span, enclosing_sp: Span) -> Span {
     }
 }
 
+// _____________________________________________________________________________
+// Loc, LocWithOpt, FileMapAndLine, FileMapAndBytePos
+//
+
 /// A source code location used for error reporting
 pub struct Loc {
     /// Information about the original source
@@ -192,6 +222,11 @@ pub struct LocWithOpt {
 pub struct FileMapAndLine { pub fm: Rc<FileMap>, pub line: usize }
 pub struct FileMapAndBytePos { pub fm: Rc<FileMap>, pub pos: BytePos }
 
+
+// _____________________________________________________________________________
+// MacroFormat, NameAndSpan, ExpnInfo, ExpnId
+//
+
 /// The syntax with which a macro was invoked.
 #[derive(Clone, Copy, Hash, Debug)]
 pub enum MacroFormat {
@@ -254,6 +289,10 @@ impl ExpnId {
     }
 }
 
+// _____________________________________________________________________________
+// FileMap, MultiByteChar, FileName, FileLines
+//
+
 pub type FileName = String;
 
 pub struct FileLines {
@@ -262,7 +301,7 @@ pub struct FileLines {
 }
 
 /// Identifies an offset of a multi-byte character in a FileMap
-#[derive(Copy)]
+#[derive(Copy, RustcEncodable, RustcDecodable, Eq, PartialEq)]
 pub struct MultiByteChar {
     /// The absolute offset of the character in the CodeMap
     pub pos: BytePos,
@@ -277,15 +316,136 @@ pub struct FileMap {
     /// e.g. `<anon>`
     pub name: FileName,
     /// The complete source code
-    pub src: String,
+    pub src: Option<Rc<String>>,
     /// The start position of this source in the CodeMap
     pub start_pos: BytePos,
+    /// The end position of this source in the CodeMap
+    pub end_pos: BytePos,
     /// Locations of lines beginnings in the source code
     pub lines: RefCell<Vec<BytePos>>,
     /// Locations of multi-byte characters in the source code
     pub multibyte_chars: RefCell<Vec<MultiByteChar>>,
 }
 
+impl Encodable for FileMap {
+    fn encode<S: Encoder>(&self, s: &mut S) -> Result<(), S::Error> {
+        s.emit_struct("FileMap", 5, |s| {
+            try! { s.emit_struct_field("name", 0, |s| self.name.encode(s)) };
+            try! { s.emit_struct_field("start_pos", 1, |s| self.start_pos.encode(s)) };
+            try! { s.emit_struct_field("end_pos", 2, |s| self.end_pos.encode(s)) };
+            try! { s.emit_struct_field("lines", 3, |s| {
+                    let lines = self.lines.borrow();
+                    // store the length
+                    try! { s.emit_u32(lines.len() as u32) };
+
+                    if lines.len() > 0 {
+                        // In order to preserve some space, we exploit the fact that
+                        // the lines list is sorted and individual lines are
+                        // probably not that long. Because of that we can store lines
+                        // as a difference list, using as little space as possible
+                        // for the differences.
+                        let max_line_length = if lines.len() == 1 {
+                            0
+                        } else {
+                            lines.as_slice()
+                                 .windows(2)
+                                 .map(|w| w[1] - w[0])
+                                 .map(|bp| bp.to_usize())
+                                 .max()
+                                 .unwrap()
+                        };
+
+                        let bytes_per_diff: u8 = match max_line_length {
+                            0 ... 0xFF => 1,
+                            0x100 ... 0xFFFF => 2,
+                            _ => 4
+                        };
+
+                        // Encode the number of bytes used per diff.
+                        try! { bytes_per_diff.encode(s) };
+
+                        // Encode the first element.
+                        try! { lines[0].encode(s) };
+
+                        let diff_iter = (&lines[..]).windows(2)
+                                                    .map(|w| (w[1] - w[0]));
+
+                        match bytes_per_diff {
+                            1 => for diff in diff_iter { try! { (diff.0 as u8).encode(s) } },
+                            2 => for diff in diff_iter { try! { (diff.0 as u16).encode(s) } },
+                            4 => for diff in diff_iter { try! { (diff.0 as u32).encode(s) } },
+                            _ => unreachable!()
+                        }
+                    }
+
+                    Ok(())
+                })
+            };
+            s.emit_struct_field("multibyte_chars", 4, |s| {
+                (*self.multibyte_chars.borrow()).encode(s)
+            })
+        })
+    }
+}
+
+impl Decodable for FileMap {
+    fn decode<D: Decoder>(d: &mut D) -> Result<FileMap, D::Error> {
+
+        d.read_struct("FileMap", 5, |d| {
+            let name: String = try! {
+                d.read_struct_field("name", 0, |d| Decodable::decode(d))
+            };
+            let start_pos: BytePos = try! {
+                d.read_struct_field("start_pos", 1, |d| Decodable::decode(d))
+            };
+            let end_pos: BytePos = try! {
+                d.read_struct_field("end_pos", 2, |d| Decodable::decode(d))
+            };
+            let lines: Vec<BytePos> = try! {
+                d.read_struct_field("lines", 3, |d| {
+                    let num_lines: u32 = try! { Decodable::decode(d) };
+                    let mut lines = Vec::with_capacity(num_lines as usize);
+
+                    if num_lines > 0 {
+                        // Read the number of bytes used per diff.
+                        let bytes_per_diff: u8 = try! { Decodable::decode(d) };
+
+                        // Read the first element.
+                        let mut line_start: BytePos = try! { Decodable::decode(d) };
+                        lines.push(line_start);
+
+                        for _ in 1..num_lines {
+                            let diff = match bytes_per_diff {
+                                1 => try! { d.read_u8() } as u32,
+                                2 => try! { d.read_u16() } as u32,
+                                4 => try! { d.read_u32() },
+                                _ => unreachable!()
+                            };
+
+                            line_start = line_start + BytePos(diff);
+
+                            lines.push(line_start);
+                        }
+                    }
+
+                    Ok(lines)
+                })
+            };
+            let multibyte_chars: Vec<MultiByteChar> = try! {
+                d.read_struct_field("multibyte_chars", 4, |d| Decodable::decode(d))
+            };
+            Ok(FileMap {
+                name: name,
+                start_pos: start_pos,
+                end_pos: end_pos,
+                src: None,
+                lines: RefCell::new(lines),
+                multibyte_chars: RefCell::new(multibyte_chars)
+            })
+        })
+    }
+}
+
 impl FileMap {
     /// EFFECT: register a start-of-line offset in the
     /// table of line-beginnings.
@@ -307,17 +467,22 @@ impl FileMap {
     /// get a line from the list of pre-computed line-beginnings
     ///
     pub fn get_line(&self, line_number: usize) -> Option<String> {
+        match self.src {
+            Some(ref src) => {
                 let lines = self.lines.borrow();
                 lines.get(line_number).map(|&line| {
                     let begin: BytePos = line - self.start_pos;
                     let begin = begin.to_usize();
-            let slice = &self.src[begin..];
+                    let slice = &src[begin..];
                     match slice.find('\n') {
                         Some(e) => &slice[..e],
                         None => slice
                     }.to_string()
                 })
             }
+            None => None
+        }
+    }
 
     pub fn record_multibyte_char(&self, pos: BytePos, bytes: usize) {
         assert!(bytes >=2 && bytes <= 4);
@@ -332,7 +497,16 @@ impl FileMap {
         !(self.name.starts_with("<") &&
           self.name.ends_with(">"))
     }
+
+    pub fn is_imported(&self) -> bool {
+        self.src.is_none()
     }
+}
+
+
+// _____________________________________________________________________________
+// CodeMap
+//
 
 pub struct CodeMap {
     pub files: RefCell<Vec<Rc<FileMap>>>,
@@ -351,7 +525,7 @@ impl CodeMap {
         let mut files = self.files.borrow_mut();
         let start_pos = match files.last() {
             None => 0,
-            Some(last) => last.start_pos.to_usize() + last.src.len(),
+            Some(last) => last.end_pos.to_usize(),
         };
 
         // Remove utf-8 BOM if any.
@@ -372,10 +546,13 @@ impl CodeMap {
             src.push('\n');
         }
 
+        let end_pos = start_pos + src.len();
+
         let filemap = Rc::new(FileMap {
             name: filename,
-            src: src.to_string(),
+            src: Some(Rc::new(src)),
             start_pos: Pos::from_usize(start_pos),
+            end_pos: Pos::from_usize(end_pos),
             lines: RefCell::new(Vec::new()),
             multibyte_chars: RefCell::new(Vec::new()),
         });
@@ -385,6 +562,45 @@ impl CodeMap {
         filemap
     }
 
+    /// Allocates a new FileMap representing a source file from an external
+    /// crate. The source code of such an "imported filemap" is not available,
+    /// but we still know enough to generate accurate debuginfo location
+    /// information for things inlined from other crates.
+    pub fn new_imported_filemap(&self,
+                                filename: FileName,
+                                source_len: usize,
+                                file_local_lines: Vec<BytePos>,
+                                file_local_multibyte_chars: Vec<MultiByteChar>)
+                                -> Rc<FileMap> {
+        let mut files = self.files.borrow_mut();
+        let start_pos = match files.last() {
+            None => 0,
+            Some(last) => last.end_pos.to_usize(),
+        };
+
+        let end_pos = Pos::from_usize(start_pos + source_len);
+        let start_pos = Pos::from_usize(start_pos);
+
+        let lines = file_local_lines.map_in_place(|pos| pos + start_pos);
+        let multibyte_chars = file_local_multibyte_chars.map_in_place(|mbc| MultiByteChar {
+            pos: mbc.pos + start_pos,
+            bytes: mbc.bytes
+        });
+
+        let filemap = Rc::new(FileMap {
+            name: filename,
+            src: None,
+            start_pos: start_pos,
+            end_pos: end_pos,
+            lines: RefCell::new(lines),
+            multibyte_chars: RefCell::new(multibyte_chars),
+        });
+
+        files.push(filemap.clone());
+
+        filemap
+    }
+
     pub fn mk_substr_filename(&self, sp: Span) -> String {
         let pos = self.lookup_char_pos(sp.lo);
         (format!("<{}:{}:{}>",
@@ -442,30 +658,42 @@ impl CodeMap {
             return Err(SpanSnippetError::IllFormedSpan(sp));
         }
 
-        let begin = self.lookup_byte_offset(sp.lo);
-        let end = self.lookup_byte_offset(sp.hi);
+        let local_begin = self.lookup_byte_offset(sp.lo);
+        let local_end = self.lookup_byte_offset(sp.hi);
 
-        if begin.fm.start_pos != end.fm.start_pos {
+        if local_begin.fm.start_pos != local_end.fm.start_pos {
             return Err(SpanSnippetError::DistinctSources(DistinctSources {
-                begin: (begin.fm.name.clone(),
-                        begin.fm.start_pos),
-                end: (end.fm.name.clone(),
-                      end.fm.start_pos)
+                begin: (local_begin.fm.name.clone(),
+                        local_begin.fm.start_pos),
+                end: (local_end.fm.name.clone(),
+                      local_end.fm.start_pos)
             }));
         } else {
-            let start = begin.pos.to_usize();
-            let limit = end.pos.to_usize();
-            if start > limit || limit > begin.fm.src.len() {
+            match local_begin.fm.src {
+                Some(ref src) => {
+                    let start_index = local_begin.pos.to_usize();
+                    let end_index = local_end.pos.to_usize();
+                    let source_len = (local_begin.fm.end_pos -
+                                      local_begin.fm.start_pos).to_usize();
+
+                    if start_index > end_index || end_index > source_len {
                         return Err(SpanSnippetError::MalformedForCodemap(
                             MalformedCodemapPositions {
-                        name: begin.fm.name.clone(),
-                        source_len: begin.fm.src.len(),
-                        begin_pos: begin.pos,
-                        end_pos: end.pos,
+                                name: local_begin.fm.name.clone(),
+                                source_len: source_len,
+                                begin_pos: local_begin.pos,
+                                end_pos: local_end.pos,
                             }));
                     }
 
-            return Ok((&begin.fm.src[start..limit]).to_string())
+                    return Ok((&src[start_index..end_index]).to_string())
+                }
+                None => {
+                    return Err(SpanSnippetError::SourceNotAvailable {
+                        filename: local_begin.fm.name.clone()
+                    });
+                }
+            }
         }
     }
 
@@ -478,6 +706,7 @@ impl CodeMap {
         panic!("asking for {} which we don't know about", filename);
     }
 
+    /// For a global BytePos compute the local offset within the containing FileMap
     pub fn lookup_byte_offset(&self, bpos: BytePos) -> FileMapAndBytePos {
         let idx = self.lookup_filemap_idx(bpos);
         let fm = (*self.files.borrow())[idx].clone();
@@ -639,11 +868,16 @@ impl CodeMap {
     }
 }
 
+// _____________________________________________________________________________
+// SpanSnippetError, DistinctSources, MalformedCodemapPositions
+//
+
 #[derive(Clone, PartialEq, Eq, Debug)]
 pub enum SpanSnippetError {
     IllFormedSpan(Span),
     DistinctSources(DistinctSources),
     MalformedForCodemap(MalformedCodemapPositions),
+    SourceNotAvailable { filename: String }
 }
 
 #[derive(Clone, PartialEq, Eq, Debug)]
@@ -660,6 +894,11 @@ pub struct MalformedCodemapPositions {
     end_pos: BytePos
 }
 
+
+// _____________________________________________________________________________
+// Tests
+//
+
 #[cfg(test)]
 mod test {
     use super::*;

src/libsyntax/parse/lexer/mod.rs

@@ -76,6 +76,10 @@ pub struct StringReader<'a> {
     // are revised to go directly to token-trees.
     /// Is \x00<name>,<ctxt>\x00 is interpreted as encoded ast::Ident?
     read_embedded_ident: bool,
+
+    // cache a direct reference to the source text, so that we don't have to
+    // retrieve it via `self.filemap.src.as_ref().unwrap()` all the time.
+    source_text: Rc<String>
 }
 
 impl<'a> Reader for StringReader<'a> {
@@ -142,6 +146,13 @@ impl<'a> StringReader<'a> {
     /// For comments.rs, which hackily pokes into pos and curr
     pub fn new_raw<'b>(span_diagnostic: &'b SpanHandler,
                        filemap: Rc<codemap::FileMap>) -> StringReader<'b> {
+        if filemap.src.is_none() {
+            span_diagnostic.handler.bug(&format!("Cannot lex filemap without source: {}",
+                                                 filemap.name)[..]);
+        }
+
+        let source_text = (*filemap.src.as_ref().unwrap()).clone();
+
         let mut sr = StringReader {
             span_diagnostic: span_diagnostic,
             pos: filemap.start_pos,
@@ -153,6 +164,7 @@ impl<'a> StringReader<'a> {
             peek_tok: token::Eof,
             peek_span: codemap::DUMMY_SP,
             read_embedded_ident: false,
+            source_text: source_text
         };
         sr.bump();
         sr
@@ -213,7 +225,7 @@ impl<'a> StringReader<'a> {
         m.push_str(": ");
         let from = self.byte_offset(from_pos).to_usize();
         let to = self.byte_offset(to_pos).to_usize();
-        m.push_str(&self.filemap.src[from..to]);
+        m.push_str(&self.source_text[from..to]);
         self.fatal_span_(from_pos, to_pos, &m[..]);
     }
 
@@ -270,8 +282,7 @@ impl<'a> StringReader<'a> {
     fn with_str_from_to<T, F>(&self, start: BytePos, end: BytePos, f: F) -> T where
         F: FnOnce(&str) -> T,
     {
-        f(&self.filemap.src[
-                self.byte_offset(start).to_usize()..
+        f(&self.source_text[self.byte_offset(start).to_usize()..
                             self.byte_offset(end).to_usize()])
     }
 
@@ -321,12 +332,10 @@ impl<'a> StringReader<'a> {
     pub fn bump(&mut self) {
         self.last_pos = self.pos;
         let current_byte_offset = self.byte_offset(self.pos).to_usize();
-        if current_byte_offset < self.filemap.src.len() {
+        if current_byte_offset < self.source_text.len() {
             assert!(self.curr.is_some());
             let last_char = self.curr.unwrap();
-            let next = self.filemap
-                          .src
-                          .char_range_at(current_byte_offset);
+            let next = self.source_text.char_range_at(current_byte_offset);
             let byte_offset_diff = next.next - current_byte_offset;
             self.pos = self.pos + Pos::from_usize(byte_offset_diff);
             self.curr = Some(next.ch);
@@ -346,8 +355,8 @@ impl<'a> StringReader<'a> {
 
     pub fn nextch(&self) -> Option<char> {
         let offset = self.byte_offset(self.pos).to_usize();
-        if offset < self.filemap.src.len() {
-            Some(self.filemap.src.char_at(offset))
+        if offset < self.source_text.len() {
+            Some(self.source_text.char_at(offset))
         } else {
             None
         }
@@ -359,7 +368,7 @@ impl<'a> StringReader<'a> {
 
     pub fn nextnextch(&self) -> Option<char> {
         let offset = self.byte_offset(self.pos).to_usize();
-        let s = &*self.filemap.src;
+        let s = &self.source_text[..];
         if offset >= s.len() { return None }
         let str::CharRange { next, .. } = s.char_range_at(offset);
         if next < s.len() {

src/libsyntax/parse/mod.rs

@@ -751,6 +751,7 @@ pub fn integer_lit(s: &str, suffix: Option<&str>, sd: &SpanHandler, sp: Span) ->
 #[cfg(test)]
 mod test {
     use super::*;
+    use std::rc::Rc;
     use serialize::json;
     use codemap::{Span, BytePos, Pos, Spanned, NO_EXPANSION};
     use owned_slice::OwnedSlice;
@@ -857,115 +858,48 @@ mod test {
     #[test]
     fn string_to_tts_1() {
         let tts = string_to_tts("fn a (b : i32) { b; }".to_string());
-        assert_eq!(json::encode(&tts).unwrap(),
-        "[\
-    {\
-        \"variant\":\"TtToken\",\
-        \"fields\":[\
-            null,\
-            {\
-                \"variant\":\"Ident\",\
-                \"fields\":[\
-                    \"fn\",\
-                    \"Plain\"\
-                ]\
-            }\
-        ]\
-    },\
-    {\
-        \"variant\":\"TtToken\",\
-        \"fields\":[\
-            null,\
-            {\
-                \"variant\":\"Ident\",\
-                \"fields\":[\
-                    \"a\",\
-                    \"Plain\"\
-                ]\
-            }\
-        ]\
-    },\
-    {\
-        \"variant\":\"TtDelimited\",\
-        \"fields\":[\
-            null,\
-            {\
-                \"delim\":\"Paren\",\
-                \"open_span\":null,\
-                \"tts\":[\
-                    {\
-                        \"variant\":\"TtToken\",\
-                        \"fields\":[\
-                            null,\
-                            {\
-                                \"variant\":\"Ident\",\
-                                \"fields\":[\
-                                    \"b\",\
-                                    \"Plain\"\
-                                ]\
-                            }\
-                        ]\
-                    },\
-                    {\
-                        \"variant\":\"TtToken\",\
-                        \"fields\":[\
-                            null,\
-                            \"Colon\"\
-                        ]\
-                    },\
-                    {\
-                        \"variant\":\"TtToken\",\
-                        \"fields\":[\
-                            null,\
-                            {\
-                                \"variant\":\"Ident\",\
-                                \"fields\":[\
-                                    \"i32\",\
-                                    \"Plain\"\
-                                ]\
-                            }\
-                        ]\
-                    }\
-                ],\
-                \"close_span\":null\
-            }\
-        ]\
-    },\
-    {\
-        \"variant\":\"TtDelimited\",\
-        \"fields\":[\
-            null,\
-            {\
-                \"delim\":\"Brace\",\
-                \"open_span\":null,\
-                \"tts\":[\
-                    {\
-                        \"variant\":\"TtToken\",\
-                        \"fields\":[\
-                            null,\
-                            {\
-                                \"variant\":\"Ident\",\
-                                \"fields\":[\
-                                    \"b\",\
-                                    \"Plain\"\
-                                ]\
-                            }\
-                        ]\
-                    },\
-                    {\
-                        \"variant\":\"TtToken\",\
-                        \"fields\":[\
-                            null,\
-                            \"Semi\"\
-                        ]\
-                    }\
-                ],\
-                \"close_span\":null\
-            }\
-        ]\
-    }\
-]"
-        );
+
+        let expected = vec![
+            ast::TtToken(sp(0, 2),
+                         token::Ident(str_to_ident("fn"),
+                         token::IdentStyle::Plain)),
+            ast::TtToken(sp(3, 4),
+                         token::Ident(str_to_ident("a"),
+                         token::IdentStyle::Plain)),
+            ast::TtDelimited(
+                sp(5, 14),
+                Rc::new(ast::Delimited {
+                    delim: token::DelimToken::Paren,
+                    open_span: sp(5, 6),
+                    tts: vec![
+                        ast::TtToken(sp(6, 7),
+                                     token::Ident(str_to_ident("b"),
+                                     token::IdentStyle::Plain)),
+                        ast::TtToken(sp(8, 9),
+                                     token::Colon),
+                        ast::TtToken(sp(10, 13),
+                                     token::Ident(str_to_ident("i32"),
+                                     token::IdentStyle::Plain)),
+                    ],
+                    close_span: sp(13, 14),
+                })),
+            ast::TtDelimited(
+                sp(15, 21),
+                Rc::new(ast::Delimited {
+                    delim: token::DelimToken::Brace,
+                    open_span: sp(15, 16),
+                    tts: vec![
+                        ast::TtToken(sp(17, 18),
+                                     token::Ident(str_to_ident("b"),
+                                     token::IdentStyle::Plain)),
+                        ast::TtToken(sp(18, 19),
+                                     token::Semi)
+                    ],
+                    close_span: sp(20, 21),
+                }))
+        ];
+
+        assert_eq!(tts, expected);
     }
 
     #[test] fn ret_expr() {

src/test/auxiliary/cross_crate_spans.rs

@@ -0,0 +1,26 @@
+// Copyright 2013-2015 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+#![crate_type = "rlib"]
+#![omit_gdb_pretty_printer_section]
+
+// no-prefer-dynamic
+// compile-flags:-g
+
+pub fn generic_function<T: Clone>(val: T) -> (T, T) {
+    let result = (val.clone(), val.clone());
+    let a_variable: u32 = 123456789;
+    let another_variable: f64 = 123456789.5;
+    zzz();
+    result
+}
+
+#[inline(never)]
+fn zzz() {()}

src/test/debuginfo/cross-crate-spans.rs

@@ -0,0 +1,74 @@
+// Copyright 2013-2015 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+#![omit_gdb_pretty_printer_section]
+
+// ignore-android: FIXME(#10381)
+// min-lldb-version: 310
+
+// aux-build:cross_crate_spans.rs
+extern crate cross_crate_spans;
+
+// compile-flags:-g
+
+
+// === GDB TESTS ===================================================================================
+
+// gdb-command:break cross_crate_spans.rs:21
+// gdb-command:run
+
+// gdb-command:print result
+// gdb-check:$1 = {17, 17}
+// gdb-command:print a_variable
+// gdb-check:$2 = 123456789
+// gdb-command:print another_variable
+// gdb-check:$3 = 123456789.5
+// gdb-command:continue
+
+// gdb-command:print result
+// gdb-check:$4 = {1212, 1212}
+// gdb-command:print a_variable
+// gdb-check:$5 = 123456789
+// gdb-command:print another_variable
+// gdb-check:$6 = 123456789.5
+// gdb-command:continue
+
+
+
+// === LLDB TESTS ==================================================================================
+
+// lldb-command:b cross_crate_spans.rs:21
+// lldb-command:run
+
+// lldb-command:print result
+// lldb-check:[...]$0 = (17, 17)
+// lldb-command:print a_variable
+// lldb-check:[...]$1 = 123456789
+// lldb-command:print another_variable
+// lldb-check:[...]$2 = 123456789.5
+// lldb-command:continue
+
+// lldb-command:print result
+// lldb-check:[...]$3 = (1212, 1212)
+// lldb-command:print a_variable
+// lldb-check:[...]$4 = 123456789
+// lldb-command:print another_variable
+// lldb-check:[...]$5 = 123456789.5
+// lldb-command:continue
+
+
+// This test makes sure that we can break in functions inlined from other crates.
+
+fn main() {
+
+    let _ = cross_crate_spans::generic_function(17u32);
+    let _ = cross_crate_spans::generic_function(1212i16);
+
+}