Auto merge of #79979 - GuillaumeGomez:rustdoc-gui-tests, r=Mark-Simulacrum

Rustdoc gui tests

This is a reopening of #70533.

For this first version, there will be no screenshot comparison. Also, a big change compared to the previous version: the tests are now hosted in the rust repository directly. Since there is no image, it's pretty lightweight to say the least.

So now, only remains the nodejs script to run the tests and the tests themselves. Just one thing is missing: where should I put the documentation for these tests? I'm not sure where would be the best place for that. The doc will contain important information like the documentation of the framework used and how to install it (`npm install browser-ui-test`, but still needs to be put somewhere so no one is lost).

We'd also need to install the package when running the CI too. For now, it runs as long as we have nodejs installed, but I think we don't it to run in all nodejs targets?

cc `@jyn514`

r? `@Mark-Simulacrum`

.gitignore

@@ -50,4 +50,6 @@ Session.vim
 .cargo
 !/src/test/run-make/thumb-none-qemu/example/.cargo
 no_llvm_build
+**node_modules
+**package-lock.json
 # Before adding new lines, see the comment at the top.

src/bootstrap/builder.rs

@@ -440,6 +440,7 @@ impl<'a> Builder<'a> {
                 test::CompiletestTest,
                 test::RustdocJSStd,
                 test::RustdocJSNotStd,
+                test::RustdocGUI,
                 test::RustdocTheme,
                 test::RustdocUi,
                 test::RustdocJson,

src/bootstrap/config.rs

@@ -174,6 +174,7 @@ pub struct Config {
     pub mandir: Option<PathBuf>,
     pub codegen_tests: bool,
     pub nodejs: Option<PathBuf>,
+    pub npm: Option<PathBuf>,
     pub gdb: Option<PathBuf>,
     pub python: Option<PathBuf>,
     pub cargo_native_static: bool,
@@ -364,6 +365,7 @@ struct Build {
     fast_submodules: Option<bool>,
     gdb: Option<String>,
     nodejs: Option<String>,
+    npm: Option<String>,
     python: Option<String>,
     locked_deps: Option<bool>,
     vendor: Option<bool>,
@@ -654,6 +656,7 @@ impl Config {
         };
 
         config.nodejs = build.nodejs.map(PathBuf::from);
+        config.npm = build.npm.map(PathBuf::from);
         config.gdb = build.gdb.map(PathBuf::from);
         config.python = build.python.map(PathBuf::from);
         set(&mut config.low_priority, build.low_priority);

src/bootstrap/lib.rs

@@ -637,6 +637,10 @@ impl Build {
         self.out.join(&*target.triple).join("doc")
     }
 
+    fn test_out(&self, target: TargetSelection) -> PathBuf {
+        self.out.join(&*target.triple).join("test")
+    }
+
     /// Output directory for all documentation for a target
     fn compiler_doc_out(&self, target: TargetSelection) -> PathBuf {
         self.out.join(&*target.triple).join("compiler-doc")

src/bootstrap/mk/Makefile.in

@@ -45,6 +45,10 @@ check-aux:
 		src/tools/cargo \
 		src/tools/cargotest \
 		$(BOOTSTRAP_ARGS)
+check-aux-and-gui: check-aux
+	$(Q)$(BOOTSTRAP) test --stage 2 \
+		src/test/rustdoc-gui \
+		$(BOOTSTRAP_ARGS)
 check-bootstrap:
 	$(Q)$(CFG_PYTHON) $(CFG_SRC_DIR)src/bootstrap/bootstrap_test.py
 dist:

src/bootstrap/sanity.rs

@@ -113,6 +113,13 @@ pub fn check(build: &mut Build) {
         .or_else(|| cmd_finder.maybe_have("node"))
         .or_else(|| cmd_finder.maybe_have("nodejs"));
 
+    build.config.npm = build
+        .config
+        .npm
+        .take()
+        .map(|p| cmd_finder.must_have(p))
+        .or_else(|| cmd_finder.maybe_have("npm"));
+
     build.config.gdb = build
         .config
         .gdb

src/bootstrap/test.rs

@@ -688,6 +688,78 @@ impl Step for RustdocJSNotStd {
     }
 }
 
+#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
+pub struct RustdocGUI {
+    pub target: TargetSelection,
+    pub compiler: Compiler,
+}
+
+impl Step for RustdocGUI {
+    type Output = ();
+    const DEFAULT: bool = true;
+    const ONLY_HOSTS: bool = true;
+
+    fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+        run.path("src/test/rustdoc-gui")
+    }
+
+    fn make_run(run: RunConfig<'_>) {
+        let compiler = run.builder.compiler(run.builder.top_stage, run.build_triple());
+        run.builder.ensure(RustdocGUI { target: run.target, compiler });
+    }
+
+    fn run(self, builder: &Builder<'_>) {
+        if let (Some(nodejs), Some(npm)) = (&builder.config.nodejs, &builder.config.npm) {
+            builder.ensure(compile::Std { compiler: self.compiler, target: self.target });
+
+            // The goal here is to check if the necessary packages are installed, and if not, we
+            // display a warning and move on.
+            let mut command = Command::new(&npm);
+            command.arg("list").arg("--depth=0");
+            let lines = command
+                .output()
+                .map(|output| String::from_utf8_lossy(&output.stdout).to_string())
+                .unwrap_or(String::new());
+            if !lines.contains(&" browser-ui-test@") {
+                println!(
+                    "warning: rustdoc-gui test suite cannot be run because npm `browser-ui-test` \
+                     dependency is missing",
+                );
+                println!(
+                    "If you want to install the `{0}` dependency, run `npm install {0}`",
+                    "browser-ui-test",
+                );
+                return;
+            }
+
+            let out_dir = builder.test_out(self.target).join("rustdoc-gui");
+            let mut command = builder.rustdoc_cmd(self.compiler);
+            command.arg("src/test/rustdoc-gui/lib.rs").arg("-o").arg(&out_dir);
+            builder.run(&mut command);
+
+            for file in fs::read_dir("src/test/rustdoc-gui").unwrap() {
+                let file = file.unwrap();
+                let file_path = file.path();
+                let file_name = file.file_name();
+
+                if !file_name.to_str().unwrap().ends_with(".goml") {
+                    continue;
+                }
+                let mut command = Command::new(&nodejs);
+                command
+                    .arg("src/tools/rustdoc-gui/tester.js")
+                    .arg("--doc-folder")
+                    .arg(out_dir.join("test_docs"))
+                    .arg("--test-file")
+                    .arg(file_path);
+                builder.run(&mut command);
+            }
+        } else {
+            builder.info("No nodejs found, skipping \"src/test/rustdoc-gui\" tests");
+        }
+    }
+}
+
 #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
 pub struct Tidy;
 
@@ -1048,6 +1120,9 @@ note: if you're sure you want to do this, please open an issue as to why. In the
         if let Some(ref nodejs) = builder.config.nodejs {
             cmd.arg("--nodejs").arg(nodejs);
         }
+        if let Some(ref npm) = builder.config.npm {
+            cmd.arg("--npm").arg(npm);
+        }
 
         let mut flags = if is_rustdoc { Vec::new() } else { vec!["-Crpath".to_string()] };
         if !is_rustdoc {

src/ci/docker/host-x86_64/x86_64-gnu-aux/Dockerfile

@@ -17,10 +17,30 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
   libgl1-mesa-dev \
   llvm-dev \
   libfreetype6-dev \
-  libexpat1-dev
+  libexpat1-dev \
+  libexpat1-dev \
+  gnupg \
+  apt-utils \
+  wget \
+  fonts-ipafont-gothic \
+  fonts-wqy-zenhei \
+  fonts-thai-tlwg \
+  fonts-kacst \
+  fonts-freefont-ttf \
+  libxss1 \
+  libxtst6
+
+RUN curl -sL https://nodejs.org/dist/v14.4.0/node-v14.4.0-linux-x64.tar.xz | tar -xJ
+ENV PATH="/node-v14.4.0-linux-x64/bin:${PATH}"
+
+# Install required dependencies from browser-UI-test framework
+# For now, we need to use `--unsafe-perm=true` to go around an issue when npm tries
+# to create a new folder. For reference:
+# https://github.com/puppeteer/puppeteer/issues/375
+RUN npm install browser-ui-test -g --unsafe-perm=true
 
 COPY scripts/sccache.sh /scripts/
 RUN sh /scripts/sccache.sh
 
 ENV RUST_CONFIGURE_ARGS --build=x86_64-unknown-linux-gnu
-ENV RUST_CHECK_TARGET check-aux
+ENV RUST_CHECK_TARGET check-aux-and-gui

src/test/rustdoc-gui/basic-code.goml

@@ -0,0 +1,3 @@
+goto: file://|DOC_PATH|/index.html
+click: ".srclink"
+assert: (".line-numbers", 1)

src/test/rustdoc-gui/basic.goml

@@ -0,0 +1,4 @@
+goto: file://|DOC_PATH|/index.html
+assert: ("#functions")
+goto: ./struct.Foo.html
+assert: ("div.type-decl")

src/test/rustdoc-gui/code-sidebar-toggle.goml

@@ -0,0 +1,6 @@
+goto: file://|DOC_PATH|/index.html
+click: ".srclink"
+click: "#sidebar-toggle"
+wait-for: 500
+fail: true
+assert: ("#source-sidebar", { "left": "-300px" })

src/test/rustdoc-gui/lib.rs

@@ -0,0 +1,67 @@
+//! The point of this crate is to be able to have enough different "kinds" of
+//! documentation generated so we can test each different features.
+
+#![crate_name = "test_docs"]
+
+use std::fmt;
+
+/// Basic function with some code examples:
+///
+/// ```
+/// println!("nothing fancy");
+/// ```
+///
+/// A failing to compile one:
+///
+/// ```compile_fail
+/// println!("where did my argument {} go? :'(");
+/// ```
+///
+/// An ignored one:
+///
+/// ```ignore (it's a test)
+/// Let's say I'm just some text will ya?
+/// ```
+pub fn foo() {}
+
+/// Just a normal struct.
+pub struct Foo;
+
+/// Just a normal enum.
+pub enum WhoLetTheDogOut {
+    /// Woof!
+    Woof,
+    /// Meoooooooow...
+    Meow,
+}
+
+/// Who doesn't love to wrap a `format!` call?
+pub fn some_more_function<T: fmt::Debug>(t: &T) -> String {
+    format!("{:?}", t)
+}
+
+/// Woohoo! A trait!
+pub trait AnotherOne {
+    /// Some func 1.
+    fn func1();
+
+    /// Some func 2.
+    fn func2();
+
+    /// Some func 3.
+    fn func3();
+}
+
+/// Check for "i" signs in lists!
+///
+/// 1. elem 1
+/// 2.test 1
+///   ```compile_fail
+///   fn foo() {}
+///   ```
+/// 3. elem 3
+/// 4. ```ignore (it's a test)
+///    fn foo() {}
+///    ```
+/// 5. elem 5
+pub fn check_list_code_block() {}

src/test/rustdoc-gui/list_code_block.goml

@@ -0,0 +1,3 @@
+goto: file://|DOC_PATH|/index.html
+goto: ./fn.check_list_code_block.html
+assert: ("pre.rust.fn")

src/test/rustdoc-gui/theme-change.goml

@@ -0,0 +1,10 @@
+goto: file://|DOC_PATH|/index.html
+click: "#theme-picker"
+click: "#theme-choices > button:first-child"
+wait-for: 500
+// should be the ayu theme so let's check the color
+assert: ("body", { "background-color": "rgb(15, 20, 25)" })
+click: "#theme-choices > button:last-child"
+wait-for: 500
+// should be the light theme so let's check the color
+assert: ("body", { "background-color": "rgb(255, 255, 255)" })

src/test/rustdoc-gui/toggle-docs.goml

@@ -0,0 +1,7 @@
+goto: file://|DOC_PATH|/index.html
+click: "#toggle-all-docs"
+wait-for: 5000
+assert: ("#main > div.docblock.hidden-by-usual-hider")
+click: "#toggle-all-docs"
+wait-for: 5000
+assert: ("#main > div.docblock.hidden-by-usual-hider", 0)

src/tools/compiletest/src/common.rs

@@ -344,6 +344,8 @@ pub struct Config {
 
     /// Path to a NodeJS executable. Used for JS doctests, emscripten and WASM tests
     pub nodejs: Option<String>,
+    /// Path to a npm executable. Used for rustdoc GUI tests
+    pub npm: Option<String>,
 }
 
 #[derive(Debug, Clone)]

src/tools/compiletest/src/main.rs

@@ -126,6 +126,7 @@ pub fn parse_config(args: Vec<String>) -> Config {
         .reqopt("", "llvm-components", "list of LLVM components built in", "LIST")
         .optopt("", "llvm-bin-dir", "Path to LLVM's `bin` directory", "PATH")
         .optopt("", "nodejs", "the name of nodejs", "PATH")
+        .optopt("", "npm", "the name of npm", "PATH")
         .optopt("", "remote-test-client", "path to the remote test client", "PATH")
         .optopt(
             "",
@@ -264,6 +265,7 @@ pub fn parse_config(args: Vec<String>) -> Config {
         linker: matches.opt_str("linker"),
         llvm_components: matches.opt_str("llvm-components").unwrap(),
         nodejs: matches.opt_str("nodejs"),
+        npm: matches.opt_str("npm"),
     }
 }
 

src/tools/compiletest/src/runtest.rs

@@ -1585,7 +1585,7 @@ impl<'test> TestCx<'test> {
 
         let aux_dir = self.aux_output_dir_name();
 
-        let rustdoc_path = self.config.rustdoc_path.as_ref().expect("--rustdoc-path passed");
+        let rustdoc_path = self.config.rustdoc_path.as_ref().expect("--rustdoc-path not passed");
         let mut rustdoc = Command::new(rustdoc_path);
 
         rustdoc

src/tools/rustdoc-gui/tester.js

@@ -0,0 +1,89 @@
+// This package needs to be install:
+//
+// ```
+// npm install browser-ui-test
+// ```
+const path = require('path');
+const {Options, runTest} = require('browser-ui-test');
+
+function showHelp() {
+    console.log("rustdoc-js options:");
+    console.log("  --doc-folder [PATH]        : location of the generated doc folder");
+    console.log("  --help                     : show this message then quit");
+    console.log("  --test-file [PATH]         : location of the JS test file");
+}
+
+function parseOptions(args) {
+    var opts = {
+        "doc_folder": "",
+        "test_file": "",
+    };
+    var correspondances = {
+        "--doc-folder": "doc_folder",
+        "--test-file": "test_file",
+    };
+
+    for (var i = 0; i < args.length; ++i) {
+        if (args[i] === "--doc-folder"
+            || args[i] === "--test-file") {
+            i += 1;
+            if (i >= args.length) {
+                console.log("Missing argument after `" + args[i - 1] + "` option.");
+                return null;
+            }
+            opts[correspondances[args[i - 1]]] = args[i];
+        } else if (args[i] === "--help") {
+            showHelp();
+            process.exit(0);
+        } else {
+            console.log("Unknown option `" + args[i] + "`.");
+            console.log("Use `--help` to see the list of options");
+            return null;
+        }
+    }
+    if (opts["test_file"].length < 1) {
+        console.log("Missing `--test-file` option.");
+    } else if (opts["doc_folder"].length < 1) {
+        console.log("Missing `--doc-folder` option.");
+    } else {
+        return opts;
+    }
+    return null;
+}
+
+function checkFile(test_file, opts, loaded, index) {
+    const test_name = path.basename(test_file, ".js");
+
+    process.stdout.write('Checking "' + test_name + '" ... ');
+    return runChecks(test_file, loaded, index);
+}
+
+function main(argv) {
+    var opts = parseOptions(argv.slice(2));
+    if (opts === null) {
+        process.exit(1);
+    }
+
+    const options = new Options();
+    try {
+        // This is more convenient that setting fields one by one.
+        options.parseArguments([
+            '--no-screenshot',
+            "--variable", "DOC_PATH", opts["doc_folder"],
+        ]);
+    } catch (error) {
+        console.error(`invalid argument: ${error}`);
+        process.exit(1);
+    }
+
+    runTest(opts["test_file"], options).then(out => {
+        const [output, nb_failures] = out;
+        console.log(output);
+        process.exit(nb_failures);
+    }).catch(err => {
+        console.error(err);
+        process.exit(1);
+    });
+}
+
+main(process.argv);