From ad9e26dab3ae8f5d739e89167338bc97b99905b7 Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Thu, 20 Feb 2014 01:14:51 -0800 Subject: [PATCH] rustdoc: Add syntax highlighting This adds simple syntax highlighting based off libsyntax's lexer to be sure to stay up to date with rust's grammar. Some of the highlighting is a bit ad-hoc, but it definitely seems to get the job done! This currently doesn't highlight rustdoc-rendered function signatures and structs that are emitted to each page because the colors already signify what's clickable and I think we'd have to figure out a different scheme before colorizing them. This does, however, colorize all code examples and source code. Closes #11393 --- src/librustdoc/html/highlight.rs | 174 ++++++++++++++++++ src/librustdoc/html/markdown.rs | 28 ++- src/librustdoc/html/render.rs | 17 +- src/librustdoc/html/static/main.css | 15 ++ src/librustdoc/lib.rs | 1 + src/libstd/fmt/mod.rs | 24 +-- src/libstd/num/mod.rs | 4 +- .../run-make/rustdoc-hidden-line/verify.sh | 2 +- 8 files changed, 239 insertions(+), 26 deletions(-) create mode 100644 src/librustdoc/html/highlight.rs diff --git a/src/librustdoc/html/highlight.rs b/src/librustdoc/html/highlight.rs new file mode 100644 index 00000000000..40b892cd9b4 --- /dev/null +++ b/src/librustdoc/html/highlight.rs @@ -0,0 +1,174 @@ +// Copyright 2014 The Rust Project Developers. See the COPYRIGHT +// file at the top-level directory of this distribution and at +// http://rust-lang.org/COPYRIGHT. +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +//! Basic html highlighting functionality +//! +//! This module uses libsyntax's lexer to provide token-based highlighting for +//! the HTML documentation generated by rustdoc. + +use std::str; +use std::io; + +use syntax::parse; +use syntax::parse::lexer; +use syntax::diagnostic; +use syntax::codemap::{BytePos, Span}; + +use html::escape::Escape; + +use t = syntax::parse::token; + +/// Highlights some source code, returning the HTML output. +pub fn highlight(src: &str) -> ~str { + let sess = parse::new_parse_sess(); + let handler = diagnostic::mk_handler(); + let span_handler = diagnostic::mk_span_handler(handler, sess.cm); + let fm = parse::string_to_filemap(sess, src.to_owned(), ~""); + + let mut out = io::MemWriter::new(); + doit(sess, + lexer::new_string_reader(span_handler, fm), + &mut out).unwrap(); + str::from_utf8_lossy(out.unwrap()).into_owned() +} + +/// Exhausts the `lexer` writing the output into `out`. +/// +/// The general structure for this method is to iterate over each token, +/// possibly giving it an HTML span with a class specifying what flavor of token +/// it's used. All source code emission is done as slices from the source map, +/// not from the tokens themselves, in order to stay true to the original +/// source. +fn doit(sess: @parse::ParseSess, lexer: lexer::StringReader, + out: &mut Writer) -> io::IoResult<()> { + use syntax::parse::lexer::Reader; + + try!(write!(out, "
\n"));
+    let mut last = BytePos(0);
+    let mut is_attribute = false;
+    let mut is_macro = false;
+    loop {
+        let next = lexer.next_token();
+        let test = if next.tok == t::EOF {lexer.pos.get()} else {next.sp.lo};
+
+        // The lexer consumes all whitespace and non-doc-comments when iterating
+        // between tokens. If this token isn't directly adjacent to our last
+        // token, then we need to emit the whitespace/comment.
+        //
+        // If the gap has any '/' characters then we consider the whole thing a
+        // comment. This will classify some whitespace as a comment, but that
+        // doesn't matter too much for syntax highlighting purposes.
+        if test > last {
+            let snip = sess.cm.span_to_snippet(Span {
+                lo: last,
+                hi: test,
+                expn_info: None,
+            }).unwrap();
+            if snip.contains("/") {
+                try!(write!(out, "{}",
+                              Escape(snip)));
+            } else {
+                try!(write!(out, "{}", Escape(snip)));
+            }
+        }
+        last = next.sp.hi;
+        if next.tok == t::EOF { break }
+
+        let klass = match next.tok {
+            // If this '&' token is directly adjacent to another token, assume
+            // that it's the address-of operator instead of the and-operator.
+            // This allows us to give all pointers their own class (~ and @ are
+            // below).
+            t::BINOP(t::AND) if lexer.peek().sp.lo == next.sp.hi => "kw-2",
+            t::AT | t::TILDE => "kw-2",
+
+            // consider this as part of a macro invocation if there was a
+            // leading identifier
+            t::NOT if is_macro => { is_macro = false; "macro" }
+
+            // operators
+            t::EQ | t::LT | t::LE | t::EQEQ | t::NE | t::GE | t::GT |
+                t::ANDAND | t::OROR | t::NOT | t::BINOP(..) | t::RARROW |
+                t::BINOPEQ(..) | t::FAT_ARROW => "op",
+
+            // miscellaneous, no highlighting
+            t::DOT | t::DOTDOT | t::DOTDOTDOT | t::COMMA | t::SEMI |
+                t::COLON | t::MOD_SEP | t::LARROW | t::DARROW | t::LPAREN |
+                t::RPAREN | t::LBRACKET | t::LBRACE | t::RBRACE |
+                t::DOLLAR => "",
+
+            // This is the start of an attribute. We're going to want to
+            // continue highlighting it as an attribute until the ending ']' is
+            // seen, so skip out early. Down below we terminate the attribute
+            // span when we see the ']'.
+            t::POUND => {
+                is_attribute = true;
+                try!(write!(out, r"\#"));
+                continue
+            }
+            t::RBRACKET => {
+                if is_attribute {
+                    is_attribute = false;
+                    try!(write!(out, "]"));
+                    continue
+                } else {
+                    ""
+                }
+            }
+
+            // text literals
+            t::LIT_CHAR(..) | t::LIT_STR(..) | t::LIT_STR_RAW(..) => "string",
+
+            // number literals
+            t::LIT_INT(..) | t::LIT_UINT(..) | t::LIT_INT_UNSUFFIXED(..) |
+                t::LIT_FLOAT(..) | t::LIT_FLOAT_UNSUFFIXED(..) => "number",
+
+            // keywords are also included in the identifier set
+            t::IDENT(ident, _is_mod_sep) => {
+                match t::get_ident(ident).get() {
+                    "ref" | "mut" => "kw-2",
+
+                    "self" => "self",
+                    "false" | "true" => "boolval",
+
+                    "Option" | "Result" => "prelude-ty",
+                    "Some" | "None" | "Ok" | "Err" => "prelude-val",
+
+                    _ if t::is_any_keyword(&next.tok) => "kw",
+                    _ => {
+                        if lexer.peek().tok == t::NOT {
+                            is_macro = true;
+                            "macro"
+                        } else {
+                            "ident"
+                        }
+                    }
+                }
+            }
+
+            t::LIFETIME(..) => "lifetime",
+            t::DOC_COMMENT(..) => "doccomment",
+            t::UNDERSCORE | t::EOF | t::INTERPOLATED(..) => "",
+        };
+
+        // as mentioned above, use the original source code instead of
+        // stringifying this token
+        let snip = sess.cm.span_to_snippet(next.sp).unwrap();
+        if klass == "" {
+            try!(write!(out, "{}", Escape(snip)));
+        } else {
+            try!(write!(out, "{}", klass,
+                          Escape(snip)));
+        }
+    }
+
+    write!(out, "
\n") +} + diff --git a/src/librustdoc/html/markdown.rs b/src/librustdoc/html/markdown.rs index af0a43efa14..378e8ca96cd 100644 --- a/src/librustdoc/html/markdown.rs +++ b/src/librustdoc/html/markdown.rs @@ -35,6 +35,8 @@ use std::str; use std::unstable::intrinsics; use std::vec; +use html::highlight; + /// A unit struct which has the `fmt::Show` trait implemented. When /// formatted, this struct will emit the HTML corresponding to the rendered /// version of the contained markdown string. @@ -95,6 +97,7 @@ extern { fn sd_markdown_free(md: *sd_markdown); fn bufnew(unit: libc::size_t) -> *buf; + fn bufputs(b: *buf, c: *libc::c_char); fn bufrelease(b: *buf); } @@ -127,7 +130,27 @@ pub fn render(w: &mut io::Writer, s: &str) -> fmt::Result { asize: text.len() as libc::size_t, unit: 0, }; - (my_opaque.dfltblk)(ob, &buf, lang, opaque); + let rendered = if lang.is_null() { + false + } else { + vec::raw::buf_as_slice((*lang).data, + (*lang).size as uint, |rlang| { + let rlang = str::from_utf8(rlang).unwrap(); + if rlang.contains("notrust") { + (my_opaque.dfltblk)(ob, &buf, lang, opaque); + true + } else { + false + } + }) + }; + + if !rendered { + let output = highlight::highlight(text).to_c_str(); + output.with_ref(|r| { + bufputs(ob, r) + }) + } }) } } @@ -181,7 +204,8 @@ pub fn find_testable_code(doc: &str, tests: &mut ::test::Collector) { vec::raw::buf_as_slice((*lang).data, (*lang).size as uint, |lang| { let s = str::from_utf8(lang).unwrap(); - (s.contains("should_fail"), s.contains("ignore")) + (s.contains("should_fail"), s.contains("ignore") || + s.contains("notrust")) }) }; if ignore { return } diff --git a/src/librustdoc/html/render.rs b/src/librustdoc/html/render.rs index 0f017a04da3..57be4e613c1 100644 --- a/src/librustdoc/html/render.rs +++ b/src/librustdoc/html/render.rs @@ -50,10 +50,10 @@ use syntax::parse::token::InternedString; use clean; use doctree; use fold::DocFolder; -use html::escape::Escape; use html::format::{VisSpace, Method, PuritySpace}; use html::layout; use html::markdown::Markdown; +use html::highlight; /// Major driving force in all rustdoc rendering. This contains information /// about where in the tree-like hierarchy rendering is occurring and controls @@ -1091,7 +1091,8 @@ fn item_module(w: &mut Writer, cx: &Context, fn item_function(w: &mut Writer, it: &clean::Item, f: &clean::Function) -> fmt::Result { - try!(write!(w, "
{vis}{purity}fn {name}{generics}{decl}
", + try!(write!(w, "
{vis}{purity}fn \
+                    {name}{generics}{decl}
", vis = VisSpace(it.visibility), purity = PuritySpace(f.purity), name = it.name.get_ref().as_slice(), @@ -1112,7 +1113,7 @@ fn item_trait(w: &mut Writer, it: &clean::Item, } // Output the trait definition - try!(write!(w, "
{}trait {}{}{} ",
+    try!(write!(w, "
{}trait {}{}{} ",
                   VisSpace(it.visibility),
                   it.name.get_ref().as_slice(),
                   t.generics,
@@ -1231,7 +1232,7 @@ fn render_method(w: &mut Writer, meth: &clean::Item) -> fmt::Result {
 
 fn item_struct(w: &mut Writer, it: &clean::Item,
                s: &clean::Struct) -> fmt::Result {
-    try!(write!(w, "
"));
+    try!(write!(w, "
"));
     try!(render_struct(w, it, Some(&s.generics), s.struct_type, s.fields,
                          s.fields_stripped, "", true));
     try!(write!(w, "
"));
@@ -1255,7 +1256,7 @@ fn item_struct(w: &mut Writer, it: &clean::Item,
 }
 
 fn item_enum(w: &mut Writer, it: &clean::Item, e: &clean::Enum) -> fmt::Result {
-    try!(write!(w, "
{}enum {}{}",
+    try!(write!(w, "
{}enum {}{}",
                   VisSpace(it.visibility),
                   it.name.get_ref().as_slice(),
                   e.generics));
@@ -1532,7 +1533,7 @@ fn render_impl(w: &mut Writer, i: &clean::Impl,
 
 fn item_typedef(w: &mut Writer, it: &clean::Item,
                 t: &clean::Typedef) -> fmt::Result {
-    try!(write!(w, "
type {}{} = {};
",
+    try!(write!(w, "
type {}{} = {};
",
                   it.name.get_ref().as_slice(),
                   t.generics,
                   t.type_));
@@ -1625,9 +1626,7 @@ impl<'a> fmt::Show for Source<'a> {
             try!(write!(fmt.buf, "{0:1$u}\n", i, cols));
         }
         try!(write!(fmt.buf, "
"));
-        try!(write!(fmt.buf, "
"));
-        try!(write!(fmt.buf, "{}", Escape(s.as_slice())));
-        try!(write!(fmt.buf, "
"));
+        try!(write!(fmt.buf, "{}", highlight::highlight(s.as_slice())));
         Ok(())
     }
 }
diff --git a/src/librustdoc/html/static/main.css b/src/librustdoc/html/static/main.css
index 8e1876bad03..6e5cdbafcd6 100644
--- a/src/librustdoc/html/static/main.css
+++ b/src/librustdoc/html/static/main.css
@@ -303,3 +303,18 @@ a {
 .stability.Locked { border-color: #0084B6; color: #00668c; }
 
 :target { background: #FDFFD3; }
+
+pre.rust .kw { color: #cc782f; }
+pre.rust .kw-2 { color: #3bbb33; }
+pre.rust .prelude-ty { color: #3bbb33; }
+pre.rust .number { color: #c13928; }
+pre.rust .self { color: #c13928; }
+pre.rust .boolval { color: #c13928; }
+pre.rust .prelude-val { color: #c13928; }
+pre.rust .op { color: #cc782f; }
+pre.rust .comment { color: #533add; }
+pre.rust .doccomment { color: #d343d0; }
+pre.rust .macro { color: #d343d0; }
+pre.rust .string { color: #c13928; }
+pre.rust .lifetime { color: #d343d0; }
+pre.rust .attribute { color: #d343d0 !important; }
diff --git a/src/librustdoc/lib.rs b/src/librustdoc/lib.rs
index c6732b5d9e9..4fb71b6710e 100644
--- a/src/librustdoc/lib.rs
+++ b/src/librustdoc/lib.rs
@@ -38,6 +38,7 @@ pub mod core;
 pub mod doctree;
 pub mod fold;
 pub mod html {
+    pub mod highlight;
     pub mod escape;
     pub mod format;
     pub mod layout;
diff --git a/src/libstd/fmt/mod.rs b/src/libstd/fmt/mod.rs
index 38fae798d5d..f7c1986fca6 100644
--- a/src/libstd/fmt/mod.rs
+++ b/src/libstd/fmt/mod.rs
@@ -82,7 +82,7 @@ function, but the `format!` macro is a syntax extension which allows it to
 leverage named parameters. Named parameters are listed at the end of the
 argument list and have the syntax:
 
-```ignore
+```notrust
 identifier '=' expression
 ```
 
@@ -107,7 +107,7 @@ and if all references to one argument do not provide a type, then the format `?`
 is used (the type's rust-representation is printed). For example, this is an
 invalid format string:
 
-```ignore
+```notrust
 {0:d} {0:s}
 ```
 
@@ -123,7 +123,7 @@ must have the type `uint`. Although a `uint` can be printed with `{:u}`, it is
 illegal to reference an argument as such. For example, this is another invalid
 format string:
 
-```ignore
+```notrust
 {:.*s} {0:u}
 ```
 
@@ -232,7 +232,7 @@ fn main() {
 There are a number of related macros in the `format!` family. The ones that are
 currently implemented are:
 
-```rust,ignore
+```ignore
 format!      // described above
 write!       // first argument is a &mut io::Writer, the destination
 writeln!     // same as write but appends a newline
@@ -276,7 +276,7 @@ references information on the stack. Under the hood, all of
 the related macros are implemented in terms of this. First
 off, some example usage is:
 
-```rust,ignore
+```ignore
 use std::fmt;
 
 # fn lol() -> T { fail!() }
@@ -334,7 +334,7 @@ This example is the equivalent of `{0:s}` essentially.
 The select method is a switch over a `&str` parameter, and the parameter *must*
 be of the type `&str`. An example of the syntax is:
 
-```ignore
+```notrust
 {0, select, male{...} female{...} other{...}}
 ```
 
@@ -353,7 +353,7 @@ The plural method is a switch statement over a `uint` parameter, and the
 parameter *must* be a `uint`. A plural method in its full glory can be specified
 as:
 
-```ignore
+```notrust
 {0, plural, offset=1 =1{...} two{...} many{...} other{...}}
 ```
 
@@ -381,7 +381,7 @@ should not be too alien. Arguments are formatted with python-like syntax,
 meaning that arguments are surrounded by `{}` instead of the C-like `%`. The
 actual grammar for the formatting syntax is:
 
-```ignore
+```notrust
 format_string :=  [ format  ] *
 format := '{' [ argument ] [ ':' format_spec ] [ ',' function_spec ] '}'
 argument := integer | identifier
@@ -896,10 +896,10 @@ impl<'a> Formatter<'a> {
     ///
     /// # Arguments
     ///
-    ///     * is_positive - whether the original integer was positive or not.
-    ///     * prefix - if the '#' character (FlagAlternate) is provided, this
-    ///       is the prefix to put in front of the number.
-    ///     * buf - the byte array that the number has been formatted into
+    /// * is_positive - whether the original integer was positive or not.
+    /// * prefix - if the '#' character (FlagAlternate) is provided, this
+    ///   is the prefix to put in front of the number.
+    /// * buf - the byte array that the number has been formatted into
     ///
     /// This function will correctly account for the flags provided as well as
     /// the minimum width. It will not take precision into account.
diff --git a/src/libstd/num/mod.rs b/src/libstd/num/mod.rs
index b23e42ad1c6..104543d4323 100644
--- a/src/libstd/num/mod.rs
+++ b/src/libstd/num/mod.rs
@@ -53,7 +53,7 @@ pub trait Zero: Add {
     ///
     /// # Laws
     ///
-    /// ~~~ignore
+    /// ~~~notrust
     /// a + 0 = a       ∀ a ∈ Self
     /// 0 + a = a       ∀ a ∈ Self
     /// ~~~
@@ -79,7 +79,7 @@ pub trait One: Mul {
     ///
     /// # Laws
     ///
-    /// ~~~ignore
+    /// ~~~notrust
     /// a * 1 = a       ∀ a ∈ Self
     /// 1 * a = a       ∀ a ∈ Self
     /// ~~~
diff --git a/src/test/run-make/rustdoc-hidden-line/verify.sh b/src/test/run-make/rustdoc-hidden-line/verify.sh
index c1d817c998d..9c905f37d31 100755
--- a/src/test/run-make/rustdoc-hidden-line/verify.sh
+++ b/src/test/run-make/rustdoc-hidden-line/verify.sh
@@ -3,6 +3,6 @@
 file="$1/doc/foo/fn.foo.html"
 
 grep -v 'invisible' $file &&
-grep '#\[deriving(Eq)\] // Bar' $file
+grep '#.*\[.*deriving.*(.*Eq.*).*\].*//.*Bar' $file
 
 exit $?