Merge pull request #2985 from phansch/riir_update_lints

update_lints rewrite: Add structure and --print-only

.gitignore

@@ -19,6 +19,7 @@ Cargo.lock
 /target
 /clippy_lints/target
 /clippy_workspace_tests/target
+/clippy_dev/target
 
 # Generated by dogfood
 /target_recur/
@@ -33,6 +34,3 @@ helper.txt
 *.iml
 .vscode
 .idea
-
-# Used by the Clippy build script
-min_version.txt

Cargo.toml

@@ -38,6 +38,11 @@ name = "clippy-driver"
 test = false
 path = "src/driver.rs"
 
+[[bin]]
+name = "clippy-dev"
+test = false
+path = "src/main.rs"
+
 [dependencies]
 # begin automatic update
 clippy_lints = { version = "0.0.212", path = "clippy_lints" }
@@ -46,6 +51,7 @@ regex = "1"
 semver = "0.9"
 
 [dev-dependencies]
+clippy_dev = { version = "0.0.1", path = "clippy_dev" }
 cargo_metadata = "0.6"
 compiletest_rs = "0.3.7"
 lazy_static = "1.0"

ci/base-tests.sh

@@ -19,6 +19,7 @@ cd clippy_workspace_tests && PATH=$PATH:~/rust/cargo/bin cargo clippy -- -D clip
 cd clippy_workspace_tests/src && PATH=$PATH:~/rust/cargo/bin cargo clippy -- -D clippy::all && cd ../..
 cd clippy_workspace_tests/subcrate && PATH=$PATH:~/rust/cargo/bin cargo clippy -- -D clippy::all && cd ../..
 cd clippy_workspace_tests/subcrate/src && PATH=$PATH:~/rust/cargo/bin cargo clippy -- -D clippy::all && cd ../../..
+cd clippy_dev && PATH=$PATH:~/rust/cargo/bin cargo clippy -- -D clippy::all && cd ..
 # test --manifest-path
 PATH=$PATH:~/rust/cargo/bin cargo clippy --manifest-path=clippy_workspace_tests/Cargo.toml -- -D clippy::all
 cd clippy_workspace_tests/subcrate && PATH=$PATH:~/rust/cargo/bin cargo clippy --manifest-path=../Cargo.toml -- -D clippy::all && cd ../..

clippy_dev/Cargo.lock

@@ -0,0 +1,224 @@
+[[package]]
+name = "aho-corasick"
+version = "0.6.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "memchr 2.0.2 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "ansi_term"
+version = "0.11.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "winapi 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "atty"
+version = "0.2.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)",
+ "termion 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "winapi 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "bitflags"
+version = "1.0.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[[package]]
+name = "clap"
+version = "2.32.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "ansi_term 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "atty 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)",
+ "bitflags 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)",
+ "strsim 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "textwrap 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "unicode-width 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)",
+ "vec_map 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "clippy_dev"
+version = "0.0.1"
+dependencies = [
+ "clap 2.32.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "itertools 0.7.8 (registry+https://github.com/rust-lang/crates.io-index)",
+ "lazy_static 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "regex 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "either"
+version = "1.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[[package]]
+name = "itertools"
+version = "0.7.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "either 1.5.0 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "lazy_static"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "version_check 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "libc"
+version = "0.2.43"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[[package]]
+name = "memchr"
+version = "2.0.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "redox_syscall"
+version = "0.1.40"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[[package]]
+name = "redox_termios"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "redox_syscall 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "regex"
+version = "1.0.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "aho-corasick 0.6.8 (registry+https://github.com/rust-lang/crates.io-index)",
+ "memchr 2.0.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "regex-syntax 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "thread_local 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)",
+ "utf8-ranges 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "regex-syntax"
+version = "0.6.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "ucd-util 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "strsim"
+version = "0.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[[package]]
+name = "termion"
+version = "1.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)",
+ "redox_syscall 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)",
+ "redox_termios 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "textwrap"
+version = "0.10.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "unicode-width 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "thread_local"
+version = "0.3.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "lazy_static 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "ucd-util"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[[package]]
+name = "unicode-width"
+version = "0.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[[package]]
+name = "utf8-ranges"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[[package]]
+name = "vec_map"
+version = "0.8.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[[package]]
+name = "version_check"
+version = "0.1.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[[package]]
+name = "winapi"
+version = "0.3.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "winapi-i686-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "winapi-x86_64-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "winapi-i686-pc-windows-gnu"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[[package]]
+name = "winapi-x86_64-pc-windows-gnu"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[metadata]
+"checksum aho-corasick 0.6.8 (registry+https://github.com/rust-lang/crates.io-index)" = "68f56c7353e5a9547cbd76ed90f7bb5ffc3ba09d4ea9bd1d8c06c8b1142eeb5a"
+"checksum ansi_term 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b"
+"checksum atty 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)" = "9a7d5b8723950951411ee34d271d99dddcc2035a16ab25310ea2c8cfd4369652"
+"checksum bitflags 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)" = "228047a76f468627ca71776ecdebd732a3423081fcf5125585bcd7c49886ce12"
+"checksum clap 2.32.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b957d88f4b6a63b9d70d5f454ac8011819c6efa7727858f458ab71c756ce2d3e"
+"checksum either 1.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3be565ca5c557d7f59e7cfcf1844f9e3033650c929c6566f511e8005f205c1d0"
+"checksum itertools 0.7.8 (registry+https://github.com/rust-lang/crates.io-index)" = "f58856976b776fedd95533137617a02fb25719f40e7d9b01c7043cd65474f450"
+"checksum lazy_static 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ca488b89a5657b0a2ecd45b95609b3e848cf1755da332a0da46e2b2b1cb371a7"
+"checksum libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)" = "76e3a3ef172f1a0b9a9ff0dd1491ae5e6c948b94479a3021819ba7d860c8645d"
+"checksum memchr 2.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "a3b4142ab8738a78c51896f704f83c11df047ff1bda9a92a661aa6361552d93d"
+"checksum redox_syscall 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)" = "c214e91d3ecf43e9a4e41e578973adeb14b474f2bee858742d127af75a0112b1"
+"checksum redox_termios 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "7e891cfe48e9100a70a3b6eb652fef28920c117d366339687bd5576160db0f76"
+"checksum regex 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)" = "67d0301b0c6804eca7e3c275119d0b01ff3b7ab9258a65709e608a66312a1025"
+"checksum regex-syntax 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)" = "747ba3b235651f6e2f67dfa8bcdcd073ddb7c243cb21c442fc12395dfcac212d"
+"checksum strsim 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "bb4f380125926a99e52bc279241539c018323fab05ad6368b56f93d9369ff550"
+"checksum termion 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "689a3bdfaab439fd92bc87df5c4c78417d3cbe537487274e9b0b2dce76e92096"
+"checksum textwrap 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)" = "307686869c93e71f94da64286f9a9524c0f308a9e1c87a583de8e9c9039ad3f6"
+"checksum thread_local 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)" = "c6b53e329000edc2b34dbe8545fd20e55a333362d0a321909685a19bd28c3f1b"
+"checksum ucd-util 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "fd2be2d6639d0f8fe6cdda291ad456e23629558d466e2789d2c3e9892bda285d"
+"checksum unicode-width 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "882386231c45df4700b275c7ff55b6f3698780a650026380e72dabe76fa46526"
+"checksum utf8-ranges 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "fd70f467df6810094968e2fce0ee1bd0e87157aceb026a8c083bcf5e25b9efe4"
+"checksum vec_map 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)" = "05c78687fb1a80548ae3250346c3db86a80a7cdd77bda190189f2d0a0987c81a"
+"checksum version_check 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "7716c242968ee87e5542f8021178248f267f295a5c4803beae8b8b7fd9bc6051"
+"checksum winapi 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "773ef9dcc5f24b7d850d0ff101e542ff24c3b090a9768e03ff889fdef41f00fd"
+"checksum winapi-i686-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
+"checksum winapi-x86_64-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"

clippy_dev/Cargo.toml

@@ -0,0 +1,10 @@
+[package]
+name = "clippy_dev"
+version = "0.0.1"
+authors = ["Philipp Hansch <dev@phansch.net>"]
+
+[dependencies]
+clap = "~2.32"
+itertools = "0.7"
+regex = "1"
+lazy_static = "1.0"

clippy_dev/src/lib.rs

@@ -0,0 +1,157 @@
+extern crate regex;
+#[macro_use]
+extern crate lazy_static;
+extern crate itertools;
+
+use regex::Regex;
+use itertools::Itertools;
+use std::collections::HashMap;
+use std::ffi::OsStr;
+use std::fs;
+use std::io::prelude::*;
+
+lazy_static! {
+    static ref DEC_CLIPPY_LINT_RE: Regex = Regex::new(r#"(?x)
+        declare_clippy_lint!\s*[\{(]\s*
+        pub\s+(?P<name>[A-Z_][A-Z_0-9]*)\s*,\s*
+        (?P<cat>[a-z_]+)\s*,\s*
+        "(?P<desc>(?:[^"\\]+|\\(?s).(?-s))*)"\s*[})]
+    "#).unwrap();
+    static ref DEC_DEPRECATED_LINT_RE: Regex = Regex::new(r#"(?x)
+        declare_deprecated_lint!\s*[{(]\s*
+        pub\s+(?P<name>[A-Z_][A-Z_0-9]*)\s*,\s*
+        "(?P<desc>(?:[^"\\]+|\\(?s).(?-s))*)"\s*[})]
+    "#).unwrap();
+    static ref NL_ESCAPE_RE: Regex = Regex::new(r#"\\\n\s*"#).unwrap();
+    pub static ref DOCS_LINK: String = "https://rust-lang-nursery.github.io/rust-clippy/master/index.html".to_string();
+}
+
+#[derive(Clone, PartialEq, Debug)]
+pub struct Lint {
+    pub name: String,
+    pub group: String,
+    pub desc: String,
+    pub deprecation: Option<String>,
+    pub module: String,
+}
+
+impl Lint {
+    pub fn new(name: &str, group: &str, desc: &str, deprecation: Option<&str>, module: &str) -> Lint {
+        Lint {
+            name: name.to_lowercase(),
+            group: group.to_string(),
+            desc: NL_ESCAPE_RE.replace(&desc.replace("\\\"", "\""), "").to_string(),
+            deprecation: deprecation.map(|d| d.to_string()),
+            module: module.to_string(),
+        }
+    }
+
+    /// Returns all non-deprecated lints
+    pub fn active_lints(lints: &[Lint]) -> impl Iterator<Item=&Lint> {
+        lints.iter().filter(|l| l.deprecation.is_none())
+    }
+
+    /// Returns the lints in a HashMap, grouped by the different lint groups
+    pub fn by_lint_group(lints: &[Lint]) -> HashMap<String, Vec<Lint>> {
+        lints.iter().map(|lint| (lint.group.to_string(), lint.clone())).into_group_map()
+    }
+}
+
+pub fn gather_all() -> impl Iterator<Item=Lint> {
+    lint_files().flat_map(|f| gather_from_file(&f))
+}
+
+fn gather_from_file(dir_entry: &fs::DirEntry) -> impl Iterator<Item=Lint> {
+    let mut file = fs::File::open(dir_entry.path()).unwrap();
+    let mut content = String::new();
+    file.read_to_string(&mut content).unwrap();
+    parse_contents(&content, dir_entry.path().file_stem().unwrap().to_str().unwrap())
+}
+
+fn parse_contents(content: &str, filename: &str) -> impl Iterator<Item=Lint> {
+    let lints = DEC_CLIPPY_LINT_RE
+        .captures_iter(content)
+        .map(|m| Lint::new(&m["name"], &m["cat"], &m["desc"], None, filename));
+    let deprecated = DEC_DEPRECATED_LINT_RE
+        .captures_iter(content)
+        .map(|m| Lint::new( &m["name"], "Deprecated", &m["desc"], Some(&m["desc"]), filename));
+    // Removing the `.collect::<Vec<Lint>>().into_iter()` causes some lifetime issues due to the map
+    lints.chain(deprecated).collect::<Vec<Lint>>().into_iter()
+}
+
+/// Collects all .rs files in the `clippy_lints/src` directory
+fn lint_files() -> impl Iterator<Item=fs::DirEntry> {
+    fs::read_dir("../clippy_lints/src")
+        .unwrap()
+        .filter_map(|f| f.ok())
+        .filter(|f| f.path().extension() == Some(OsStr::new("rs")))
+}
+
+#[test]
+fn test_parse_contents() {
+    let result: Vec<Lint> = parse_contents(
+        r#"
+declare_clippy_lint! {
+    pub PTR_ARG,
+    style,
+    "really long \
+     text"
+}
+
+declare_clippy_lint!{
+    pub DOC_MARKDOWN,
+    pedantic,
+    "single line"
+}
+
+/// some doc comment
+declare_deprecated_lint! {
+    pub SHOULD_ASSERT_EQ,
+    "`assert!()` will be more flexible with RFC 2011"
+}
+    "#,
+    "module_name").collect();
+
+    let expected = vec![
+        Lint::new("ptr_arg", "style", "really long text", None, "module_name"),
+        Lint::new("doc_markdown", "pedantic", "single line", None, "module_name"),
+        Lint::new(
+            "should_assert_eq",
+            "Deprecated",
+            "`assert!()` will be more flexible with RFC 2011",
+            Some("`assert!()` will be more flexible with RFC 2011"),
+            "module_name"
+        ),
+    ];
+    assert_eq!(expected, result);
+}
+
+#[test]
+fn test_active_lints() {
+    let lints = vec![
+        Lint::new("should_assert_eq", "Deprecated", "abc", Some("Reason"), "module_name"),
+        Lint::new("should_assert_eq2", "Not Deprecated", "abc", None, "module_name")
+    ];
+    let expected = vec![
+        Lint::new("should_assert_eq2", "Not Deprecated", "abc", None, "module_name")
+    ];
+    assert_eq!(expected, Lint::active_lints(&lints).cloned().collect::<Vec<Lint>>());
+}
+
+#[test]
+fn test_by_lint_group() {
+    let lints = vec![
+        Lint::new("should_assert_eq", "group1", "abc", None, "module_name"),
+        Lint::new("should_assert_eq2", "group2", "abc", None, "module_name"),
+        Lint::new("incorrect_match", "group1", "abc", None, "module_name"),
+    ];
+    let mut expected: HashMap<String, Vec<Lint>> = HashMap::new();
+    expected.insert("group1".to_string(), vec![
+        Lint::new("should_assert_eq", "group1", "abc", None, "module_name"),
+        Lint::new("incorrect_match", "group1", "abc", None, "module_name"),
+    ]);
+    expected.insert("group2".to_string(), vec![
+        Lint::new("should_assert_eq2", "group2", "abc", None, "module_name")
+    ]);
+    assert_eq!(expected, Lint::by_lint_group(&lints));
+}

clippy_dev/src/main.rs

@@ -0,0 +1,45 @@
+extern crate clap;
+extern crate clippy_dev;
+extern crate regex;
+
+use clap::{App, Arg, SubCommand};
+use clippy_dev::*;
+
+fn main() {
+    let matches = App::new("Clippy developer tooling")
+        .subcommand(
+            SubCommand::with_name("update_lints")
+                .about("Update the lint list")
+                .arg(
+                    Arg::with_name("print-only")
+                        .long("print-only")
+                        .short("p")
+                        .help("Print a table of lints to STDOUT. Does not modify any files."),
+                )
+        )
+        .get_matches();
+
+    if let Some(matches) = matches.subcommand_matches("update_lints") {
+        if matches.is_present("print-only") {
+            print_lints();
+        }
+    }
+}
+
+fn print_lints() {
+    let lint_list = gather_all().collect::<Vec<Lint>>();
+    let grouped_by_lint_group = Lint::by_lint_group(&lint_list);
+
+    for (lint_group, mut lints) in grouped_by_lint_group {
+        if lint_group == "Deprecated" { continue; }
+        println!("\n## {}", lint_group);
+
+        lints.sort_by(|a, b| a.name.cmp(&b.name));
+
+        for lint in lints {
+            println!("* [{}]({}#{}) ({})", lint.name, clippy_dev::DOCS_LINK.clone(), lint.name, lint.desc);
+        }
+    }
+
+    println!("there are {} lints", Lint::active_lints(&lint_list).count());
+}

util/dev

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+cd clippy_dev && cargo run -- $@

util/update_lints.py

@@ -7,6 +7,7 @@
 import os
 import re
 import sys
+from subprocess import call
 
 declare_deprecated_lint_re = re.compile(r'''
     declare_deprecated_lint! \s* [{(] \s*
@@ -166,19 +167,7 @@ def main(print_only=False, check=False):
         all_lints += value
 
     if print_only:
-        print_clippy_lint_groups = [
+        call(["./util/dev", "update_lints", "--print-only"])
-            "correctness",
-            "style",
-            "complexity",
-            "perf",
-            "pedantic",
-            "nursery",
-            "restriction"
-        ]
-        for group in print_clippy_lint_groups:
-            sys.stdout.write('\n## ' + group + '\n')
-            for (_, name, _, descr) in sorted(clippy_lints[group]):
-                sys.stdout.write('* [' + name + '](https://rust-lang-nursery.github.io/rust-clippy/master/index.html#' + name + ') (' + descr + ')\n')
         return
 
     # update the lint counter in README.md