From c2d59067fb9f365e302710005b5c3ccc3ea85f9a Mon Sep 17 00:00:00 2001 From: est31 Date: Mon, 12 Jun 2017 21:35:47 +0200 Subject: [PATCH] Autogenerate stubs and the summary of the unstable book --- src/Cargo.lock | 7 + src/Cargo.toml | 1 + src/bootstrap/doc.rs | 27 +++- src/bootstrap/lib.rs | 5 + src/bootstrap/step.rs | 21 ++- .../src/language-features/asm.md | 2 +- .../{global_asm.md => global-asm.md} | 0 ...s_bytes.md => from-utf8-error-as-bytes.md} | 0 src/tools/tidy/src/features.rs | 66 ++++++-- src/tools/tidy/src/unstable_book.rs | 46 +++--- src/tools/unstable-book-gen/Cargo.toml | 9 ++ src/tools/unstable-book-gen/src/SUMMARY.md | 8 + src/tools/unstable-book-gen/src/main.rs | 149 ++++++++++++++++++ src/tools/unstable-book-gen/src/stub-issue.md | 7 + .../unstable-book-gen/src/stub-no-issue.md | 5 + 15 files changed, 308 insertions(+), 45 deletions(-) rename src/doc/unstable-book/src/language-features/{global_asm.md => global-asm.md} (100%) rename src/doc/unstable-book/src/library-features/{from_utf8_error_as_bytes.md => from-utf8-error-as-bytes.md} (100%) create mode 100644 src/tools/unstable-book-gen/Cargo.toml create mode 100644 src/tools/unstable-book-gen/src/SUMMARY.md create mode 100644 src/tools/unstable-book-gen/src/main.rs create mode 100644 src/tools/unstable-book-gen/src/stub-issue.md create mode 100644 src/tools/unstable-book-gen/src/stub-no-issue.md diff --git a/src/Cargo.lock b/src/Cargo.lock index efbbe36c981..80e01baa582 100644 --- a/src/Cargo.lock +++ b/src/Cargo.lock @@ -1892,6 +1892,13 @@ dependencies = [ "void 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "unstable-book-gen" +version = "0.1.0" +dependencies = [ + "tidy 0.1.0", +] + [[package]] name = "url" version = "1.4.0" diff --git a/src/Cargo.toml b/src/Cargo.toml index 85a6df3573a..8efa484287a 100644 --- a/src/Cargo.toml +++ b/src/Cargo.toml @@ -9,6 +9,7 @@ members = [ "tools/error_index_generator", "tools/linkchecker", "tools/rustbook", + "tools/unstable-book-gen", "tools/tidy", "tools/build-manifest", "tools/remote-test-client", diff --git a/src/bootstrap/doc.rs b/src/bootstrap/doc.rs index baee1ada508..30f631ca2df 100644 --- a/src/bootstrap/doc.rs +++ b/src/bootstrap/doc.rs @@ -27,18 +27,26 @@ use {Build, Compiler, Mode}; use util::{cp_r, symlink_dir}; use build_helper::up_to_date; -/// Invoke `rustbook` as compiled in `stage` for `target` for the doc book -/// `name` into the `out` path. +/// Invoke `rustbook` for `target` for the doc book `name`. /// /// This will not actually generate any documentation if the documentation has /// already been generated. pub fn rustbook(build: &Build, target: &str, name: &str) { + let src = build.src.join("src/doc"); + rustbook_src(build, target, name, &src); +} + +/// Invoke `rustbook` for `target` for the doc book `name` from the `src` path. +/// +/// This will not actually generate any documentation if the documentation has +/// already been generated. +pub fn rustbook_src(build: &Build, target: &str, name: &str, src: &Path) { let out = build.doc_out(target); t!(fs::create_dir_all(&out)); let out = out.join(name); let compiler = Compiler::new(0, &build.config.build); - let src = build.src.join("src/doc").join(name); + let src = src.join(name); let index = out.join("index.html"); let rustbook = build.tool(&compiler, "rustbook"); if up_to_date(&src, &index) && up_to_date(&rustbook, &index) { @@ -354,6 +362,19 @@ pub fn error_index(build: &Build, target: &str) { build.run(&mut index); } +pub fn unstable_book_gen(build: &Build, target: &str) { + println!("Generating unstable book md files ({})", target); + let out = build.md_doc_out(target).join("unstable-book"); + t!(fs::create_dir_all(&out)); + t!(fs::remove_dir_all(&out)); + let compiler = Compiler::new(0, &build.config.build); + let mut cmd = build.tool_cmd(&compiler, "unstable-book-gen"); + cmd.arg(build.src.join("src")); + cmd.arg(out); + + build.run(&mut cmd); +} + fn symlink_dir_force(src: &Path, dst: &Path) -> io::Result<()> { if let Ok(m) = fs::symlink_metadata(dst) { if m.file_type().is_dir() { diff --git a/src/bootstrap/lib.rs b/src/bootstrap/lib.rs index 2fe6a2a3ae8..1c59debddbb 100644 --- a/src/bootstrap/lib.rs +++ b/src/bootstrap/lib.rs @@ -677,6 +677,11 @@ impl Build { self.out.join(target).join("doc") } + /// Output directory for some generated md crate documentation for a target (temporary) + fn md_doc_out(&self, target: &str) -> PathBuf { + self.out.join(target).join("md-doc") + } + /// Output directory for all crate documentation for a target (temporary) /// /// The artifacts here are then copied into `doc_out` above. diff --git a/src/bootstrap/step.rs b/src/bootstrap/step.rs index 9e8b08a23b7..684a00ce7f1 100644 --- a/src/bootstrap/step.rs +++ b/src/bootstrap/step.rs @@ -548,6 +548,10 @@ pub fn build_rules<'a>(build: &'a Build) -> Rules { .dep(|s| s.name("maybe-clean-tools")) .dep(|s| s.name("librustc-tool")) .run(move |s| compile::tool(build, s.stage, s.target, "error_index_generator")); + rules.build("tool-unstable-book-gen", "src/tools/unstable-book-gen") + .dep(|s| s.name("maybe-clean-tools")) + .dep(|s| s.name("libstd-tool")) + .run(move |s| compile::tool(build, s.stage, s.target, "unstable-book-gen")); rules.build("tool-tidy", "src/tools/tidy") .dep(|s| s.name("maybe-clean-tools")) .dep(|s| s.name("libstd-tool")) @@ -662,8 +666,17 @@ pub fn build_rules<'a>(build: &'a Build) -> Rules { .target(&build.config.build) .stage(0) }) + .dep(move |s| { + s.name("doc-unstable-book-gen") + .host(&build.config.build) + .target(&build.config.build) + .stage(0) + }) .default(build.config.docs) - .run(move |s| doc::rustbook(build, s.target, "unstable-book")); + .run(move |s| doc::rustbook_src(build, + s.target, + "unstable-book", + &build.md_doc_out(s.target))); rules.doc("doc-standalone", "src/doc") .dep(move |s| { s.name("rustc") @@ -679,6 +692,12 @@ pub fn build_rules<'a>(build: &'a Build) -> Rules { .default(build.config.docs) .host(true) .run(move |s| doc::error_index(build, s.target)); + rules.doc("doc-unstable-book-gen", "src/tools/unstable-book-gen") + .dep(move |s| s.name("tool-unstable-book-gen").target(&build.config.build).stage(0)) + .dep(move |s| s.name("librustc-link")) + .default(build.config.docs) + .host(true) + .run(move |s| doc::unstable_book_gen(build, s.target)); for (krate, path, default) in krates("std") { rules.doc(&krate.doc_step, path) .dep(|s| s.name("libstd-link")) diff --git a/src/doc/unstable-book/src/language-features/asm.md b/src/doc/unstable-book/src/language-features/asm.md index 8deb8f46256..f22095fe5de 100644 --- a/src/doc/unstable-book/src/language-features/asm.md +++ b/src/doc/unstable-book/src/language-features/asm.md @@ -190,4 +190,4 @@ constraints, etc. [llvm-docs]: http://llvm.org/docs/LangRef.html#inline-assembler-expressions If you need more power and don't mind losing some of the niceties of -`asm!`, check out [global_asm](language-features/global_asm.html). +`asm!`, check out [global_asm](language-features/global-asm.html). diff --git a/src/doc/unstable-book/src/language-features/global_asm.md b/src/doc/unstable-book/src/language-features/global-asm.md similarity index 100% rename from src/doc/unstable-book/src/language-features/global_asm.md rename to src/doc/unstable-book/src/language-features/global-asm.md diff --git a/src/doc/unstable-book/src/library-features/from_utf8_error_as_bytes.md b/src/doc/unstable-book/src/library-features/from-utf8-error-as-bytes.md similarity index 100% rename from src/doc/unstable-book/src/library-features/from_utf8_error_as_bytes.md rename to src/doc/unstable-book/src/library-features/from-utf8-error-as-bytes.md diff --git a/src/tools/tidy/src/features.rs b/src/tools/tidy/src/features.rs index e34821e3584..81db23ccceb 100644 --- a/src/tools/tidy/src/features.rs +++ b/src/tools/tidy/src/features.rs @@ -24,7 +24,7 @@ use std::fs::File; use std::io::prelude::*; use std::path::Path; -#[derive(Debug, PartialEq)] +#[derive(Debug, PartialEq, Clone)] pub enum Status { Stable, Removed, @@ -42,13 +42,16 @@ impl fmt::Display for Status { } } -#[derive(Debug)] +#[derive(Debug, Clone)] pub struct Feature { pub level: Status, pub since: String, pub has_gate_test: bool, + pub tracking_issue: Option, } +pub type Features = HashMap; + pub fn check(path: &Path, bad: &mut bool, quiet: bool) { let mut features = collect_lang_features(path); assert!(!features.is_empty()); @@ -168,8 +171,7 @@ fn find_attr_val<'a>(line: &'a str, attr: &str) -> Option<&'a str> { .map(|(i, j)| &line[i..j]) } -fn test_filen_gate(filen_underscore: &str, - features: &mut HashMap) -> bool { +fn test_filen_gate(filen_underscore: &str, features: &mut Features) -> bool { if filen_underscore.starts_with("feature_gate") { for (n, f) in features.iter_mut() { if filen_underscore == format!("feature_gate_{}", n) { @@ -181,7 +183,7 @@ fn test_filen_gate(filen_underscore: &str, return false; } -pub fn collect_lang_features(base_src_path: &Path) -> HashMap { +pub fn collect_lang_features(base_src_path: &Path) -> Features { let mut contents = String::new(); let path = base_src_path.join("libsyntax/feature_gate.rs"); t!(t!(File::open(path)).read_to_string(&mut contents)); @@ -197,11 +199,19 @@ pub fn collect_lang_features(base_src_path: &Path) -> HashMap { }; let name = parts.next().unwrap().trim(); let since = parts.next().unwrap().trim().trim_matches('"'); + let issue_str = parts.next().unwrap().trim(); + let tracking_issue = if issue_str.starts_with("None") { + None + } else { + let s = issue_str.split("(").nth(1).unwrap().split(")").nth(0).unwrap(); + Some(s.parse().unwrap()) + }; Some((name.to_owned(), Feature { - level: level, + level, since: since.to_owned(), has_gate_test: false, + tracking_issue, })) }) .collect() @@ -209,8 +219,8 @@ pub fn collect_lang_features(base_src_path: &Path) -> HashMap { pub fn collect_lib_features(base_src_path: &Path, bad: &mut bool, - features: &HashMap) -> HashMap { - let mut lib_features = HashMap::::new(); + features: &Features) -> Features { + let mut lib_features = Features::new(); let mut contents = String::new(); super::walk(base_src_path, &mut |path| super::filter_dirs(path) || path.ends_with("src/test"), @@ -224,10 +234,32 @@ pub fn collect_lib_features(base_src_path: &Path, contents.truncate(0); t!(t!(File::open(&file), &file).read_to_string(&mut contents)); + let mut becoming_feature: Option<(String, Feature)> = None; for (i, line) in contents.lines().enumerate() { let mut err = |msg: &str| { tidy_error!(bad, "{}:{}: {}", file.display(), i + 1, msg); }; + if let Some((ref name, ref mut f)) = becoming_feature { + if f.tracking_issue.is_none() { + f.tracking_issue = find_attr_val(line, "issue") + .map(|s| s.parse().unwrap()); + } + if line.ends_with("]") { + lib_features.insert(name.to_owned(), f.clone()); + } else if !line.ends_with(",") && !line.ends_with("\\") { + // We need to bail here because we might have missed the + // end of a stability attribute above because the "]" + // might not have been at the end of the line. + // We could then get into the very unfortunate situation that + // we continue parsing the file assuming the current stability + // attribute has not ended, and ignoring possible feature + // attributes in the process. + err("malformed stability attribute"); + } else { + continue; + } + } + becoming_feature = None; let level = if line.contains("[unstable(") { Status::Unstable } else if line.contains("[stable(") { @@ -250,6 +282,7 @@ pub fn collect_lib_features(base_src_path: &Path, } None => "None", }; + let tracking_issue = find_attr_val(line, "issue").map(|s| s.parse().unwrap()); if features.contains_key(feature_name) { err("duplicating a lang feature"); @@ -263,12 +296,17 @@ pub fn collect_lib_features(base_src_path: &Path, } continue; } - lib_features.insert(feature_name.to_owned(), - Feature { - level: level, - since: since.to_owned(), - has_gate_test: false, - }); + let feature = Feature { + level, + since: since.to_owned(), + has_gate_test: false, + tracking_issue, + }; + if line.contains("]") { + lib_features.insert(feature_name.to_owned(), feature); + } else { + becoming_feature = Some((feature_name.to_owned(), feature)); + } } }); lib_features diff --git a/src/tools/tidy/src/unstable_book.rs b/src/tools/tidy/src/unstable_book.rs index 5a6524b3e88..e05ab470eed 100644 --- a/src/tools/tidy/src/unstable_book.rs +++ b/src/tools/tidy/src/unstable_book.rs @@ -11,26 +11,28 @@ use std::collections::HashSet; use std::fs; use std::path; -use features::{collect_lang_features, collect_lib_features, Status}; +use features::{collect_lang_features, collect_lib_features, Features, Status}; -const PATH_STR: &'static str = "doc/unstable-book/src"; +pub const PATH_STR: &str = "doc/unstable-book/src"; -const LANG_FEATURES_DIR: &'static str = "language-features"; +pub const COMPILER_FLAGS_DIR: &str = "compiler-flags"; -const LIB_FEATURES_DIR: &'static str = "library-features"; +pub const LANG_FEATURES_DIR: &str = "language-features"; + +pub const LIB_FEATURES_DIR: &str = "library-features"; /// Build the path to the Unstable Book source directory from the Rust 'src' directory -fn unstable_book_path(base_src_path: &path::Path) -> path::PathBuf { +pub fn unstable_book_path(base_src_path: &path::Path) -> path::PathBuf { base_src_path.join(PATH_STR) } /// Directory where the features are documented within the Unstable Book source directory -fn unstable_book_lang_features_path(base_src_path: &path::Path) -> path::PathBuf { +pub fn unstable_book_lang_features_path(base_src_path: &path::Path) -> path::PathBuf { unstable_book_path(base_src_path).join(LANG_FEATURES_DIR) } /// Directory where the features are documented within the Unstable Book source directory -fn unstable_book_lib_features_path(base_src_path: &path::Path) -> path::PathBuf { +pub fn unstable_book_lib_features_path(base_src_path: &path::Path) -> path::PathBuf { unstable_book_path(base_src_path).join(LIB_FEATURES_DIR) } @@ -42,27 +44,16 @@ fn dir_entry_is_file(dir_entry: &fs::DirEntry) -> bool { .is_file() } -/// Retrieve names of all lang-related unstable features -fn collect_unstable_lang_feature_names(base_src_path: &path::Path) -> HashSet { - collect_lang_features(base_src_path) - .into_iter() +/// Retrieve names of all unstable features +pub fn collect_unstable_feature_names(features: &Features) -> HashSet { + features + .iter() .filter(|&(_, ref f)| f.level == Status::Unstable) - .map(|(ref name, _)| name.to_owned()) + .map(|(name, _)| name.to_owned()) .collect() } -/// Retrieve names of all lib-related unstable features -fn collect_unstable_lib_feature_names(base_src_path: &path::Path) -> HashSet { - let mut bad = true; - let lang_features = collect_lang_features(base_src_path); - collect_lib_features(base_src_path, &mut bad, &lang_features) - .into_iter() - .filter(|&(_, ref f)| f.level == Status::Unstable) - .map(|(ref name, _)| name.to_owned()) - .collect() -} - -fn collect_unstable_book_section_file_names(dir: &path::Path) -> HashSet { +pub fn collect_unstable_book_section_file_names(dir: &path::Path) -> HashSet { fs::read_dir(dir) .expect("could not read directory") .into_iter() @@ -95,7 +86,10 @@ pub fn check(path: &path::Path, bad: &mut bool) { // Library features - let unstable_lib_feature_names = collect_unstable_lib_feature_names(path); + let lang_features = collect_lang_features(path); + let lib_features = collect_lib_features(path, bad, &lang_features); + + let unstable_lib_feature_names = collect_unstable_feature_names(&lib_features); let unstable_book_lib_features_section_file_names = collect_unstable_book_lib_features_section_file_names(path); @@ -119,7 +113,7 @@ pub fn check(path: &path::Path, bad: &mut bool) { // Language features - let unstable_lang_feature_names = collect_unstable_lang_feature_names(path); + let unstable_lang_feature_names = collect_unstable_feature_names(&lang_features); let unstable_book_lang_features_section_file_names = collect_unstable_book_lang_features_section_file_names(path); diff --git a/src/tools/unstable-book-gen/Cargo.toml b/src/tools/unstable-book-gen/Cargo.toml new file mode 100644 index 00000000000..4751a5e4151 --- /dev/null +++ b/src/tools/unstable-book-gen/Cargo.toml @@ -0,0 +1,9 @@ +[package] +authors = ["est31 ", + "The Rust Project Developers"] +name = "unstable-book-gen" +version = "0.1.0" +license = "MIT/Apache-2.0" + +[dependencies] +tidy = { path = "../tidy" } diff --git a/src/tools/unstable-book-gen/src/SUMMARY.md b/src/tools/unstable-book-gen/src/SUMMARY.md new file mode 100644 index 00000000000..933c928e2f0 --- /dev/null +++ b/src/tools/unstable-book-gen/src/SUMMARY.md @@ -0,0 +1,8 @@ +[The Unstable Book](the-unstable-book.md) + +- [Compiler flags](compiler-flags.md) +{compiler_flags} +- [Language features](language-features.md) +{language_features} +- [Library Features](library-features.md) +{library_features} diff --git a/src/tools/unstable-book-gen/src/main.rs b/src/tools/unstable-book-gen/src/main.rs new file mode 100644 index 00000000000..adec73d4a69 --- /dev/null +++ b/src/tools/unstable-book-gen/src/main.rs @@ -0,0 +1,149 @@ +// Copyright 2017 The Rust Project Developers. See the COPYRIGHT +// file at the top-level directory of this distribution and at +// http://rust-lang.org/COPYRIGHT. +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +//! Auto-generate stub docs for the unstable book + +#![deny(warnings)] + +extern crate tidy; + +use tidy::features::{Feature, Features, collect_lib_features, collect_lang_features}; +use tidy::unstable_book::{collect_unstable_feature_names, collect_unstable_book_section_file_names, + PATH_STR, LANG_FEATURES_DIR, LIB_FEATURES_DIR}; +use std::collections::HashSet; +use std::io::Write; +use std::fs::{self, File}; +use std::env; +use std::path::Path; + +/// A helper macro to `unwrap` a result except also print out details like: +/// +/// * The file/line of the panic +/// * The expression that failed +/// * The error itself +macro_rules! t { + ($e:expr) => (match $e { + Ok(e) => e, + Err(e) => panic!("{} failed with {}", stringify!($e), e), + }) +} + +fn generate_stub_issue(path: &Path, name: &str, issue: u32) { + let mut file = t!(File::create(path)); + t!(file.write_fmt(format_args!(include_str!("stub-issue.md"), + name = name, + issue = issue))); +} + +fn generate_stub_no_issue(path: &Path, name: &str) { + let mut file = t!(File::create(path)); + t!(file.write_fmt(format_args!(include_str!("stub-no-issue.md"), + name = name))); +} + +fn hset_to_summary_str(hset: HashSet, dir: &str +) -> String { + hset + .iter() + .map(|ref n| format!(" - [{}]({}/{}.md)", + n, + dir, + n.replace('_', "-"))) + .fold("".to_owned(), |s, a| s + &a + "\n") +} + +fn generate_summary(path: &Path, lang_features: &Features, lib_features: &Features) { + let compiler_flags = collect_unstable_book_section_file_names( + &path.join("compiler-flags")); + + let compiler_flags_str = hset_to_summary_str(compiler_flags, + "compiler-flags"); + + let unstable_lang_features = collect_unstable_feature_names(&lang_features); + let unstable_lib_features = collect_unstable_feature_names(&lib_features); + + let lang_features_str = hset_to_summary_str(unstable_lang_features, + LANG_FEATURES_DIR); + let lib_features_str = hset_to_summary_str(unstable_lib_features, + LIB_FEATURES_DIR); + + let mut file = t!(File::create(&path.join("SUMMARY.md"))); + t!(file.write_fmt(format_args!(include_str!("SUMMARY.md"), + compiler_flags = compiler_flags_str, + language_features = lang_features_str, + library_features = lib_features_str))); + +} + +fn has_valid_tracking_issue(f: &Feature) -> bool { + if let Some(n) = f.tracking_issue { + if n > 0 { + return true; + } + } + false +} + +fn generate_unstable_book_files(src :&Path, out: &Path, features :&Features) { + let unstable_features = collect_unstable_feature_names(features); + let unstable_section_file_names = collect_unstable_book_section_file_names(src); + t!(fs::create_dir_all(&out)); + for feature_name in &unstable_features - &unstable_section_file_names { + let file_name = format!("{}.md", feature_name.replace('_', "-")); + let out_file_path = out.join(&file_name); + let feature = &features[&feature_name]; + + if has_valid_tracking_issue(&feature) { + generate_stub_issue(&out_file_path, &feature_name, feature.tracking_issue.unwrap()); + } else { + generate_stub_no_issue(&out_file_path, &feature_name); + } + } +} + +fn copy_recursive(path: &Path, to: &Path) { + for entry in t!(fs::read_dir(path)) { + let e = t!(entry); + let t = t!(e.metadata()); + let dest = &to.join(e.file_name()); + if t.is_file() { + t!(fs::copy(&e.path(), dest)); + } else if t.is_dir() { + t!(fs::create_dir_all(dest)); + copy_recursive(&e.path(), dest); + } + } +} + +fn main() { + let src_path_str = env::args_os().skip(1).next().expect("source path required"); + let dest_path_str = env::args_os().skip(2).next().expect("destination path required"); + let src_path = Path::new(&src_path_str); + let dest_path = Path::new(&dest_path_str).join("src"); + + let lang_features = collect_lang_features(src_path); + let mut bad = false; + let lib_features = collect_lib_features(src_path, &mut bad, &lang_features); + + let doc_src_path = src_path.join(PATH_STR); + + t!(fs::create_dir_all(&dest_path)); + + generate_unstable_book_files(&doc_src_path.join(LANG_FEATURES_DIR), + &dest_path.join(LANG_FEATURES_DIR), + &lang_features); + generate_unstable_book_files(&doc_src_path.join(LIB_FEATURES_DIR), + &dest_path.join(LIB_FEATURES_DIR), + &lib_features); + + copy_recursive(&doc_src_path, &dest_path); + + generate_summary(&dest_path, &lang_features, &lib_features); +} diff --git a/src/tools/unstable-book-gen/src/stub-issue.md b/src/tools/unstable-book-gen/src/stub-issue.md new file mode 100644 index 00000000000..8698fb7278f --- /dev/null +++ b/src/tools/unstable-book-gen/src/stub-issue.md @@ -0,0 +1,7 @@ +# `{name}` + +The tracking issue for this feature is: [#{issue}] + +[#{issue}]: https://github.com/rust-lang/rust/issues/{issue} + +------------------------ diff --git a/src/tools/unstable-book-gen/src/stub-no-issue.md b/src/tools/unstable-book-gen/src/stub-no-issue.md new file mode 100644 index 00000000000..3da140633d0 --- /dev/null +++ b/src/tools/unstable-book-gen/src/stub-no-issue.md @@ -0,0 +1,5 @@ +# `{name}` + +This feature has no tracking issue, and is therefore likely internal to the compiler, not being intended for general use. + +------------------------