From cbf8342efe8acface3c2751e347212ca582ed25d Mon Sep 17 00:00:00 2001 From: Guillaume Gomez Date: Fri, 14 Apr 2017 01:23:14 +0200 Subject: [PATCH 1/3] Hoedown big comeback! --- .gitmodules | 4 + src/librustdoc/Cargo.toml | 1 + src/librustdoc/build.rs | 30 +++++ src/librustdoc/html/markdown.rs | 191 ++++++++++++++++++++++++++++++++ src/librustdoc/markdown.rs | 3 +- src/librustdoc/test.rs | 34 +++++- src/rt/hoedown | 1 + 7 files changed, 257 insertions(+), 7 deletions(-) create mode 100644 src/librustdoc/build.rs create mode 160000 src/rt/hoedown diff --git a/.gitmodules b/.gitmodules index 4f29cef8570..9003f0750a2 100644 --- a/.gitmodules +++ b/.gitmodules @@ -5,6 +5,10 @@ [submodule "src/compiler-rt"] path = src/compiler-rt url = https://github.com/rust-lang/compiler-rt.git +[submodule "src/rt/hoedown"] + path = src/rt/hoedown + url = https://github.com/rust-lang/hoedown.git + branch = rust-2015-09-21-do-not-delete [submodule "src/jemalloc"] path = src/jemalloc url = https://github.com/rust-lang/jemalloc.git diff --git a/src/librustdoc/Cargo.toml b/src/librustdoc/Cargo.toml index 52f5d99838d..d1c98a5c6f1 100644 --- a/src/librustdoc/Cargo.toml +++ b/src/librustdoc/Cargo.toml @@ -2,6 +2,7 @@ authors = ["The Rust Project Developers"] name = "rustdoc" version = "0.0.0" +build = "build.rs" [lib] name = "rustdoc" diff --git a/src/librustdoc/build.rs b/src/librustdoc/build.rs new file mode 100644 index 00000000000..4189e3d2ac7 --- /dev/null +++ b/src/librustdoc/build.rs @@ -0,0 +1,30 @@ +// Copyright 2015 The Rust Project Developers. See the COPYRIGHT +// file at the top-level directory of this distribution and at +// http://rust-lang.org/COPYRIGHT. +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +extern crate build_helper; +extern crate gcc; + +fn main() { + let src_dir = std::path::Path::new("../rt/hoedown/src"); + build_helper::rerun_if_changed_anything_in_dir(src_dir); + let mut cfg = gcc::Config::new(); + cfg.file("../rt/hoedown/src/autolink.c") + .file("../rt/hoedown/src/buffer.c") + .file("../rt/hoedown/src/document.c") + .file("../rt/hoedown/src/escape.c") + .file("../rt/hoedown/src/html.c") + .file("../rt/hoedown/src/html_blocks.c") + .file("../rt/hoedown/src/html_smartypants.c") + .file("../rt/hoedown/src/stack.c") + .file("../rt/hoedown/src/version.c") + .include(src_dir) + .compile("libhoedown.a"); +} + diff --git a/src/librustdoc/html/markdown.rs b/src/librustdoc/html/markdown.rs index c59101cc779..b02b60531d1 100644 --- a/src/librustdoc/html/markdown.rs +++ b/src/librustdoc/html/markdown.rs @@ -25,6 +25,9 @@ #![allow(non_camel_case_types)] +use libc; +use std::slice; + use std::ascii::AsciiExt; use std::cell::RefCell; use std::collections::{HashMap, VecDeque}; @@ -357,6 +360,194 @@ impl<'a, I: Iterator>> Iterator for Footnotes<'a, I> { } } +const DEF_OUNIT: libc::size_t = 64; +const HOEDOWN_EXT_NO_INTRA_EMPHASIS: libc::c_uint = 1 << 11; +const HOEDOWN_EXT_TABLES: libc::c_uint = 1 << 0; +const HOEDOWN_EXT_FENCED_CODE: libc::c_uint = 1 << 1; +const HOEDOWN_EXT_AUTOLINK: libc::c_uint = 1 << 3; +const HOEDOWN_EXT_STRIKETHROUGH: libc::c_uint = 1 << 4; +const HOEDOWN_EXT_SUPERSCRIPT: libc::c_uint = 1 << 8; +const HOEDOWN_EXT_FOOTNOTES: libc::c_uint = 1 << 2; + +const HOEDOWN_EXTENSIONS: libc::c_uint = + HOEDOWN_EXT_NO_INTRA_EMPHASIS | HOEDOWN_EXT_TABLES | + HOEDOWN_EXT_FENCED_CODE | HOEDOWN_EXT_AUTOLINK | + HOEDOWN_EXT_STRIKETHROUGH | HOEDOWN_EXT_SUPERSCRIPT | + HOEDOWN_EXT_FOOTNOTES; + +enum hoedown_document {} + +type blockcodefn = extern "C" fn(*mut hoedown_buffer, *const hoedown_buffer, + *const hoedown_buffer, *const hoedown_renderer_data, + libc::size_t); + +type blockquotefn = extern "C" fn(*mut hoedown_buffer, *const hoedown_buffer, + *const hoedown_renderer_data, libc::size_t); + +type headerfn = extern "C" fn(*mut hoedown_buffer, *const hoedown_buffer, + libc::c_int, *const hoedown_renderer_data, + libc::size_t); + +type blockhtmlfn = extern "C" fn(*mut hoedown_buffer, *const hoedown_buffer, + *const hoedown_renderer_data, libc::size_t); + +type codespanfn = extern "C" fn(*mut hoedown_buffer, *const hoedown_buffer, + *const hoedown_renderer_data, libc::size_t) -> libc::c_int; + +type linkfn = extern "C" fn (*mut hoedown_buffer, *const hoedown_buffer, + *const hoedown_buffer, *const hoedown_buffer, + *const hoedown_renderer_data, libc::size_t) -> libc::c_int; + +type entityfn = extern "C" fn (*mut hoedown_buffer, *const hoedown_buffer, + *const hoedown_renderer_data, libc::size_t); + +type normaltextfn = extern "C" fn(*mut hoedown_buffer, *const hoedown_buffer, + *const hoedown_renderer_data, libc::size_t); + +#[repr(C)] +struct hoedown_renderer_data { + opaque: *mut libc::c_void, +} + +#[repr(C)] +struct hoedown_renderer { + opaque: *mut libc::c_void, + + blockcode: Option, + blockquote: Option, + header: Option, + + other_block_level_callbacks: [libc::size_t; 11], + + blockhtml: Option, + + /* span level callbacks - NULL or return 0 prints the span verbatim */ + autolink: libc::size_t, // unused + codespan: Option, + other_span_level_callbacks_1: [libc::size_t; 7], + link: Option, + other_span_level_callbacks_2: [libc::size_t; 6], + + /* low level callbacks - NULL copies input directly into the output */ + entity: Option, + normal_text: Option, + + /* header and footer */ + other_callbacks: [libc::size_t; 2], +} + +#[repr(C)] +struct hoedown_html_renderer_state { + opaque: *mut libc::c_void, + toc_data: html_toc_data, + flags: libc::c_uint, + link_attributes: Option, +} + +#[repr(C)] +struct html_toc_data { + header_count: libc::c_int, + current_level: libc::c_int, + level_offset: libc::c_int, + nesting_level: libc::c_int, +} + +#[repr(C)] +struct hoedown_buffer { + data: *const u8, + size: libc::size_t, + asize: libc::size_t, + unit: libc::size_t, +} + +extern { + fn hoedown_html_renderer_new(render_flags: libc::c_uint, + nesting_level: libc::c_int) + -> *mut hoedown_renderer; + fn hoedown_html_renderer_free(renderer: *mut hoedown_renderer); + + fn hoedown_document_new(rndr: *const hoedown_renderer, + extensions: libc::c_uint, + max_nesting: libc::size_t) -> *mut hoedown_document; + fn hoedown_document_render(doc: *mut hoedown_document, + ob: *mut hoedown_buffer, + document: *const u8, + doc_size: libc::size_t); + fn hoedown_document_free(md: *mut hoedown_document); + + fn hoedown_buffer_new(unit: libc::size_t) -> *mut hoedown_buffer; + fn hoedown_buffer_free(b: *mut hoedown_buffer); +} + +impl hoedown_buffer { + fn as_bytes(&self) -> &[u8] { + unsafe { slice::from_raw_parts(self.data, self.size as usize) } + } +} + +pub fn old_find_testable_code(doc: &str, tests: &mut ::test::Collector, position: Span) { + extern fn block(_ob: *mut hoedown_buffer, + text: *const hoedown_buffer, + lang: *const hoedown_buffer, + data: *const hoedown_renderer_data, + line: libc::size_t) { + unsafe { + if text.is_null() { return } + let block_info = if lang.is_null() { + LangString::all_false() + } else { + let lang = (*lang).as_bytes(); + let s = str::from_utf8(lang).unwrap(); + LangString::parse(s) + }; + if !block_info.rust { return } + let opaque = (*data).opaque as *mut hoedown_html_renderer_state; + let tests = &mut *((*opaque).opaque as *mut ::test::Collector); + let line = tests.get_line() + line; + let filename = tests.get_filename(); + tests.add_old_test(line, filename); + } + } + + extern fn header(_ob: *mut hoedown_buffer, + text: *const hoedown_buffer, + level: libc::c_int, data: *const hoedown_renderer_data, + _: libc::size_t) { + unsafe { + let opaque = (*data).opaque as *mut hoedown_html_renderer_state; + let tests = &mut *((*opaque).opaque as *mut ::test::Collector); + if text.is_null() { + tests.register_header("", level as u32); + } else { + let text = (*text).as_bytes(); + let text = str::from_utf8(text).unwrap(); + tests.register_header(text, level as u32); + } + } + } + + tests.set_position(position); + + unsafe { + let ob = hoedown_buffer_new(DEF_OUNIT); + let renderer = hoedown_html_renderer_new(0, 0); + (*renderer).blockcode = Some(block); + (*renderer).header = Some(header); + (*((*renderer).opaque as *mut hoedown_html_renderer_state)).opaque + = tests as *mut _ as *mut libc::c_void; + + let document = hoedown_document_new(renderer, HOEDOWN_EXTENSIONS, 16); + hoedown_document_render(document, ob, doc.as_ptr(), + doc.len() as libc::size_t); + hoedown_document_free(document); + + hoedown_html_renderer_free(renderer); + hoedown_buffer_free(ob); + } +} + pub fn find_testable_code(doc: &str, tests: &mut ::test::Collector, position: Span) { tests.set_position(position); diff --git a/src/librustdoc/markdown.rs b/src/librustdoc/markdown.rs index 5fadda030a4..f75144c23ac 100644 --- a/src/librustdoc/markdown.rs +++ b/src/librustdoc/markdown.rs @@ -25,7 +25,7 @@ use externalfiles::{ExternalHtml, LoadStringError, load_string}; use html::render::reset_ids; use html::escape::Escape; use html::markdown; -use html::markdown::{Markdown, MarkdownWithToc, find_testable_code}; +use html::markdown::{Markdown, MarkdownWithToc, find_testable_code, old_find_testable_code}; use test::{TestOptions, Collector}; /// Separate any lines at the start of the file that begin with `# ` or `%`. @@ -159,6 +159,7 @@ pub fn test(input: &str, cfgs: Vec, libs: SearchPaths, externs: Externs, let mut collector = Collector::new(input.to_string(), cfgs, libs, externs, true, opts, maybe_sysroot, None, Some(input.to_owned())); + old_find_testable_code(&input_str, &mut collector, DUMMY_SP); find_testable_code(&input_str, &mut collector, DUMMY_SP); test_args.insert(0, "rustdoctest".to_string()); testing::test_main(&test_args, collector.tests); diff --git a/src/librustdoc/test.rs b/src/librustdoc/test.rs index f6b7a07bdae..fb681b20065 100644 --- a/src/librustdoc/test.rs +++ b/src/librustdoc/test.rs @@ -380,6 +380,8 @@ fn partition_source(s: &str) -> (String, String) { pub struct Collector { pub tests: Vec, + // to be removed when hoedown will be definitely gone + pub old_tests: Vec, names: Vec, cfgs: Vec, libs: SearchPaths, @@ -401,6 +403,7 @@ impl Collector { codemap: Option>, filename: Option) -> Collector { Collector { tests: Vec::new(), + old_tests: Vec::new(), names: Vec::new(), cfgs: cfgs, libs: libs, @@ -417,11 +420,8 @@ impl Collector { } } - pub fn add_test(&mut self, test: String, - should_panic: bool, no_run: bool, should_ignore: bool, - as_test_harness: bool, compile_fail: bool, error_codes: Vec, - line: usize, filename: String) { - let name = if self.use_headers { + fn generate_name(&self, line: usize, filename: &str) -> String { + if self.use_headers { if let Some(ref header) = self.current_header { format!("{} - {} (line {})", filename, header, line) } else { @@ -429,7 +429,27 @@ impl Collector { } } else { format!("{} - {} (line {})", filename, self.names.join("::"), line) - }; + } + } + + pub fn add_old_test(&mut self, line: usize, filename: String) { + let name = self.generate_name(line, &filename); + self.old_tests.push(name); + } + + pub fn add_test(&mut self, test: String, + should_panic: bool, no_run: bool, should_ignore: bool, + as_test_harness: bool, compile_fail: bool, error_codes: Vec, + line: usize, filename: String) { + let name = self.generate_name(line, &filename); + if self.old_tests.iter().find(|&x| x == &name).is_none() { + let _ = writeln!(&mut io::stderr(), + "WARNING: {} Code block is not currently run as a test, but will in \ + future versions of rustdoc. Please ensure this code block is a \ + runnable test, or use the `ignore` directive.", + name); + return + } let cfgs = self.cfgs.clone(); let libs = self.libs.clone(); let externs = self.externs.clone(); @@ -544,6 +564,8 @@ impl<'a, 'hir> HirCollector<'a, 'hir> { attrs.unindent_doc_comments(); if let Some(doc) = attrs.doc_value() { self.collector.cnt = 0; + markdown::old_find_testable_code(doc, self.collector, + attrs.span.unwrap_or(DUMMY_SP)); markdown::find_testable_code(doc, self.collector, attrs.span.unwrap_or(DUMMY_SP)); } diff --git a/src/rt/hoedown b/src/rt/hoedown new file mode 160000 index 00000000000..da282f1bb72 --- /dev/null +++ b/src/rt/hoedown @@ -0,0 +1 @@ +Subproject commit da282f1bb7277b4d30fa1599ee29ad8eb4dd2a92 From 695354af9c76a88da07190299c95d9c867a51cfd Mon Sep 17 00:00:00 2001 From: Guillaume Gomez Date: Fri, 14 Apr 2017 13:03:48 +0200 Subject: [PATCH 2/3] Remove hoedown from tidy check --- src/tools/tidy/src/main.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/tools/tidy/src/main.rs b/src/tools/tidy/src/main.rs index 17f8b62117a..44063e627a3 100644 --- a/src/tools/tidy/src/main.rs +++ b/src/tools/tidy/src/main.rs @@ -86,6 +86,7 @@ fn filter_dirs(path: &Path) -> bool { "src/rust-installer", "src/liblibc", "src/vendor", + "src/rt/hoedown", ]; skip.iter().any(|p| path.ends_with(p)) } From bee02913207ec24913ca4eeb007b402146ed1532 Mon Sep 17 00:00:00 2001 From: Guillaume Gomez Date: Fri, 14 Apr 2017 17:17:06 +0200 Subject: [PATCH 3/3] Add hoedown COPYRIGHT back --- COPYRIGHT | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/COPYRIGHT b/COPYRIGHT index 19559fa2950..abe89980308 100644 --- a/COPYRIGHT +++ b/COPYRIGHT @@ -197,6 +197,28 @@ their own copyright notices and license terms: USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +* Hoedown, the markdown parser, under src/rt/hoedown, is + licensed as follows. + + Copyright (c) 2008, Natacha Porté + Copyright (c) 2011, Vicent Martí + Copyright (c) 2013, Devin Torres and the Hoedown authors + + Permission to use, copy, modify, and distribute this + software for any purpose with or without fee is hereby + granted, provided that the above copyright notice and + this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR + DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE + INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND + FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY + SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR + ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA + OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE + OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN + CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + * libbacktrace, under src/libbacktrace: Copyright (C) 2012-2014 Free Software Foundation, Inc.