feat: make wikilink extraction and checking opt-in (#1803)

Author: JayJayArr

README.md

@@ -18,6 +18,7 @@ Available as a command-line utility, a library and a [GitHub Action](https://git
 
 <!-- START doctoc generated TOC please keep comment here to allow auto update -->
 <!-- DON'T EDIT THIS SECTION, INSTEAD RE-RUN doctoc TO UPDATE -->
+
 ## Table of Contents
 
 - [Development](#development)
@@ -171,7 +172,7 @@ outdated information.
 | Language             | Rust    | Ruby          | Go       | JS                    | TypeScript   | Python               | JS                    | PHP    |
 | Async/Parallel       | ![yes]  | ![yes]        | ![yes]   | ![yes]                | ![yes]       | ![yes]               | ![yes]                | ![yes] |
 | JSON output          | ![yes]  | ![no]         | ![yes]   | ![yes]                | ![yes]       | ![maybe]<sup>1</sup> | ![yes]                | ![yes] |
-| Static binary        | ![yes]  | ![no]         | ![yes]   | ![no]                 | ![no]        | ️![no]                | ![no]                 | ![no]  |
+| Static binary        | ![yes]  | ![no]         | ![yes]   | ![no]                 | ![no]        | ️![no]               | ![no]                 | ![no]  |
 | Markdown files       | ![yes]  | ![yes]        | ![no]    | ![no]                 | ![no]        | ![yes]               | ![yes]                | ![no]  |
 | HTML files           | ![yes]  | ![no]         | ![no]    | ![yes]                | ![yes]       | ![no]                | ![yes]                | ![no]  |
 | Text files           | ![yes]  | ![no]         | ![no]    | ![no]                 | ![no]        | ![no]                | ![no]                 | ![no]  |
@@ -183,7 +184,7 @@ outdated information.
 | Relative URLs        | ![yes]  | ![yes]        | ![no]    | ![yes]                | ![yes]       | ![yes]               | ![yes]                | ![yes] |
 | Anchors/Fragments    | ![yes]  | ![no]         | ![no]    | ![no]                 | ![no]        | ![yes]               | ![yes]                | ![no]  |
 | Skip relative URLs   | ![yes]  | ![no]         | ![no]    | ![maybe]              | ![no]        | ![no]                | ![no]                 | ![no]  |
-| Include patterns     | ![yes]️  | ![yes]        | ![no]    | ![yes]                | ![no]        | ![no]                | ![no]                 | ![no]  |
+| Include patterns     | ![yes]️ | ![yes]        | ![no]    | ![yes]                | ![no]        | ![no]                | ![no]                 | ![no]  |
 | Exclude patterns     | ![yes]  | ![no]         | ![yes]   | ![yes]                | ![yes]       | ![yes]               | ![yes]                | ![yes] |
 | Handle redirects     | ![yes]  | ![yes]        | ![yes]   | ![yes]                | ![yes]       | ![yes]               | ![yes]                | ![yes] |
 | Ignore insecure SSL  | ![yes]  | ![yes]        | ![yes]   | ![no]                 | ![no]        | ![yes]               | ![no]                 | ![yes] |
@@ -587,6 +588,9 @@ Options:
       --cookie-jar <COOKIE_JAR>
           Tell lychee to read cookies from the given file. Cookies will be stored in the cookie jar and sent with requests. New cookies will be stored in the cookie jar and existing cookies will be updated
 
+      --include-wikilinks
+          Check WikiLinks in Markdown files
+
   -h, --help
           Print help (see a summary with '-h')
 
@@ -710,6 +714,7 @@ which includes usage instructions.
 ## Pre-commit Usage
 
 Lychee can also be used as a [pre-commit](https://pre-commit.com/) hook.
+
 ```yaml
 # .pre-commit-config.yaml
 repos:
@@ -722,6 +727,7 @@ repos:
 ```
 
 Rather than running on staged-files only, Lychee can be run against an entire repository.
+
 ```yaml
 - id: lychee
   args: ["--no-progress", "."]

fixtures/TEST_WIKI.md

@@ -0,0 +1,2 @@
+Test file including a normal link and a wikilink
+[[LycheeWikilink]]

lychee-bin/src/main.rs

@@ -338,7 +338,9 @@ async fn run(opts: &LycheeOptions) -> Result<i32> {
         .headers(HeaderMap::from_header_pairs(&opts.config.header)?)
         .excluded_paths(PathExcludes::new(opts.config.exclude_path.clone())?)
         // File a bug if you rely on this envvar! It's going to go away eventually.
-        .use_html5ever(std::env::var("LYCHEE_USE_HTML5EVER").is_ok_and(|x| x == "1"));
+        .use_html5ever(std::env::var("LYCHEE_USE_HTML5EVER").is_ok_and(|x| x == "1"))
+        .include_wikilinks(opts.config.include_wikilinks);
+
     collector = if let Some(ref basic_auth) = opts.config.basic_auth {
         collector.basic_auth_extractor(BasicAuthExtractor::new(basic_auth)?)
     } else {

lychee-bin/src/options.rs

@@ -734,6 +734,12 @@ separated list of accepted status codes. This example will accept 200, 201,
     #[arg(long)]
     #[serde(default)]
     pub(crate) cookie_jar: Option<PathBuf>,
+
+    #[allow(clippy::doc_markdown)]
+    /// Check WikiLinks in Markdown files
+    #[arg(long)]
+    #[serde(default)]
+    pub(crate) include_wikilinks: bool,
 }
 
 impl Config {
@@ -789,6 +795,7 @@ impl Config {
             include_fragments: false;
             include_mail: false;
             include_verbatim: false;
+            include_wikilinks: false;
             include: Vec::<String>::new();
             insecure: false;
             max_cache_age: humantime::parse_duration(DEFAULT_MAX_CACHE_AGE).unwrap();

lychee-bin/tests/cli.rs

@@ -2442,6 +2442,31 @@ mod cli {
             .stdout(contains("https://www.example.com/smth."));
     }
 
+    #[test]
+    fn test_wikilink_extract_when_specified() {
+        let test_path = fixtures_path().join("TEST_WIKI.md");
+
+        let mut cmd = main_command();
+        cmd.arg("--dump")
+            .arg("--include-wikilinks")
+            .arg(test_path)
+            .assert()
+            .success()
+            .stdout(contains("LycheeWikilink"));
+    }
+
+    #[test]
+    fn test_wikilink_dont_extract_when_not_specified() {
+        let test_path = fixtures_path().join("TEST_WIKI.md");
+
+        let mut cmd = main_command();
+        cmd.arg("--dump")
+            .arg(test_path)
+            .assert()
+            .success()
+            .stdout(is_empty());
+    }
+
     #[test]
     fn test_index_files_default() {
         let input = fixtures_path().join("filechecker/dir_links.md");

lychee-lib/src/collector.rs

@@ -29,6 +29,7 @@ pub struct Collector {
     skip_ignored: bool,
     skip_hidden: bool,
     include_verbatim: bool,
+    include_wikilinks: bool,
     use_html5ever: bool,
     root_dir: Option<PathBuf>,
     base: Option<Base>,
@@ -47,6 +48,7 @@ impl Default for Collector {
             basic_auth_extractor: None,
             skip_missing_inputs: false,
             include_verbatim: false,
+            include_wikilinks: false,
             use_html5ever: false,
             skip_hidden: true,
             skip_ignored: true,
@@ -76,6 +78,7 @@ impl Collector {
             basic_auth_extractor: None,
             skip_missing_inputs: false,
             include_verbatim: false,
+            include_wikilinks: false,
             use_html5ever: false,
             skip_hidden: true,
             skip_ignored: true,
@@ -138,6 +141,14 @@ impl Collector {
         self
     }
 
+    #[allow(clippy::doc_markdown)]
+    /// Check WikiLinks in Markdown files
+    #[must_use]
+    pub const fn include_wikilinks(mut self, yes: bool) -> Self {
+        self.include_wikilinks = yes;
+        self
+    }
+
     /// Pass a [`BasicAuthExtractor`] which is capable to match found
     /// URIs to basic auth credentials. These credentials get passed to the
     /// request in question.
@@ -268,7 +279,11 @@ impl Collector {
                 let basic_auth_extractor = self.basic_auth_extractor.clone();
                 async move {
                     let content = content?;
-                    let extractor = Extractor::new(self.use_html5ever, self.include_verbatim);
+                    let extractor = Extractor::new(
+                        self.use_html5ever,
+                        self.include_verbatim,
+                        self.include_wikilinks,
+                    );
                     let uris: Vec<RawUri> = extractor.extract(&content);
                     let requests = request::create(
                         uris,

lychee-lib/src/extract/markdown.rs

@@ -14,7 +14,11 @@ fn md_extensions() -> Options {
 }
 
 /// Extract unparsed URL strings from a Markdown string.
-pub(crate) fn extract_markdown(input: &str, include_verbatim: bool) -> Vec<RawUri> {
+pub(crate) fn extract_markdown(
+    input: &str,
+    include_verbatim: bool,
+    include_wikilinks: bool,
+) -> Vec<RawUri> {
     // In some cases it is undesirable to extract links from within code blocks,
     // which is why we keep track of entries and exits while traversing the input.
     let mut inside_code_block = false;
@@ -64,6 +68,10 @@ pub(crate) fn extract_markdown(input: &str, include_verbatim: bool) -> Vec<RawUr
                      Some(extract_raw_uri_from_plaintext(&dest_url)),
                     // Wiki URL (`[[http://example.com]]`)
                     LinkType::WikiLink { has_pothole: _ } => {
+                        // Exclude WikiLinks if not explicitly enabled
+                        if !include_wikilinks {
+                            return None;
+                        }
                         inside_link_block = true;
                         //Ignore gitlab toc notation: https://docs.gitlab.com/user/markdown/#table-of-contents
                         if ["_TOC_".to_string(), "TOC".to_string()].contains(&dest_url.to_string()) {
@@ -280,7 +288,7 @@ or inline like `https://bar.org` for instance.
             },
         ];
 
-        let uris = extract_markdown(MD_INPUT, false);
+        let uris = extract_markdown(MD_INPUT, false, false);
         assert_eq!(uris, expected);
     }
 
@@ -309,7 +317,7 @@ or inline like `https://bar.org` for instance.
             },
         ];
 
-        let uris = extract_markdown(MD_INPUT, true);
+        let uris = extract_markdown(MD_INPUT, true, false);
         assert_eq!(uris, expected);
     }
 
@@ -325,7 +333,7 @@ Some pre-formatted http://pre.com
 
         let expected = vec![];
 
-        let uris = extract_markdown(input, false);
+        let uris = extract_markdown(input, false, false);
         assert_eq!(uris, expected);
     }
 
@@ -358,15 +366,15 @@ $$
 [\psi](\mathbf{L})
 $$
 ";
-        let uris = extract_markdown(input, true);
+        let uris = extract_markdown(input, true, false);
         assert!(uris.is_empty());
     }
 
     #[test]
     fn test_single_word_footnote_is_not_detected_as_link() {
         let markdown = "This footnote is[^actually] a link.\n\n[^actually]: not";
         let expected = vec![];
-        let uris = extract_markdown(markdown, true);
+        let uris = extract_markdown(markdown, true, false);
         assert_eq!(uris, expected);
     }
 
@@ -378,7 +386,7 @@ $$
             element: None,
             attribute: None,
         }];
-        let uris = extract_markdown(markdown, true);
+        let uris = extract_markdown(markdown, true, false);
         assert_eq!(uris, expected);
     }
 
@@ -390,7 +398,7 @@ $$
             element: None,
             attribute: None,
         }];
-        let uris = extract_markdown(markdown, true);
+        let uris = extract_markdown(markdown, true, false);
         assert_eq!(uris, expected);
     }
 
@@ -402,7 +410,7 @@ $$
             element: Some("a".to_string()),
             attribute: Some("href".to_string()),
         }];
-        let uris = extract_markdown(markdown, true);
+        let uris = extract_markdown(markdown, true, true);
         assert_eq!(uris, expected);
     }
 
@@ -421,14 +429,14 @@ $$
                 attribute: Some("href".to_string()),
             },
         ];
-        let uris = extract_markdown(markdown, true);
+        let uris = extract_markdown(markdown, true, true);
         assert_eq!(uris, expected);
     }
 
     #[test]
     fn test_ignore_gitlab_toc() {
         let markdown = r"[[_TOC_]][TOC]";
-        let uris = extract_markdown(markdown, true);
+        let uris = extract_markdown(markdown, true, true);
         assert!(uris.is_empty());
     }
 }

lychee-lib/src/extract/mod.rs

@@ -14,6 +14,7 @@ use plaintext::extract_raw_uri_from_plaintext;
 pub struct Extractor {
     use_html5ever: bool,
     include_verbatim: bool,
+    include_wikilinks: bool,
 }
 
 impl Extractor {
@@ -30,10 +31,11 @@ impl Extractor {
     ///   For more information, consult the `pulldown_cmark` documentation about code blocks
     ///   [here](https://docs.rs/pulldown-cmark/latest/pulldown_cmark/enum.CodeBlockKind.html)
     #[must_use]
-    pub const fn new(use_html5ever: bool, include_verbatim: bool) -> Self {
+    pub const fn new(use_html5ever: bool, include_verbatim: bool, include_wikilinks: bool) -> Self {
         Self {
             use_html5ever,
             include_verbatim,
+            include_wikilinks,
         }
     }
 
@@ -42,7 +44,11 @@ impl Extractor {
     #[must_use]
     pub fn extract(&self, input_content: &InputContent) -> Vec<RawUri> {
         match input_content.file_type {
-            FileType::Markdown => extract_markdown(&input_content.content, self.include_verbatim),
+            FileType::Markdown => extract_markdown(
+                &input_content.content,
+                self.include_verbatim,
+                self.include_wikilinks,
+            ),
             FileType::Html => {
                 if self.use_html5ever {
                     html::html5ever::extract_html(&input_content.content, self.include_verbatim)
@@ -72,7 +78,7 @@ mod tests {
     fn extract_uris(input: &str, file_type: FileType) -> HashSet<Uri> {
         let input_content = InputContent::from_string(input, file_type);
 
-        let extractor = Extractor::new(false, false);
+        let extractor = Extractor::new(false, false, false);
         let uris_html5gum: HashSet<Uri> = extractor
             .extract(&input_content)
             .into_iter()
@@ -84,7 +90,7 @@ mod tests {
             uris
         };
 
-        let extractor = Extractor::new(true, false);
+        let extractor = Extractor::new(true, false, false);
         let uris_html5ever: HashSet<Uri> = extractor
             .extract(&input_content)
             .into_iter()
@@ -216,7 +222,7 @@ mod tests {
         };
 
         for use_html5ever in [true, false] {
-            let extractor = Extractor::new(use_html5ever, false);
+            let extractor = Extractor::new(use_html5ever, false, false);
             let links = extractor.extract(input_content);
 
             let urls = links