Fix remaining isSuspiciousURL checks

Author: yshym

modules/web-discovery-project/sources/web-discovery-project.es

@@ -961,7 +961,9 @@ const WebDiscoveryProject = {
     if (clean_url != url) {
       // they are different, sanity checks
       if (
-        sanitizeUrl(clean_url, { testMode: WebDiscoveryProject.testMode }).result !== "safe" ||
+        sanitizeUrl(clean_url, {
+          testMode: WebDiscoveryProject.testMode,
+        }).result !== "safe" ||
         WebDiscoveryProject.dropLongURL(clean_url)
       )
         return url;
@@ -1025,7 +1027,10 @@ const WebDiscoveryProject = {
     }
 
     // the url is suspicious, this should never be the case here but better safe
-    if (sanitizeUrl(url, { testMode: WebDiscoveryProject.testMode }).result !== "safe") {
+    if (
+      sanitizeUrl(url, { testMode: WebDiscoveryProject.testMode }).result !==
+      "safe"
+    ) {
       return discard("URL failed the isSuspiciousURL check");
     }
 
@@ -1724,8 +1729,9 @@ const WebDiscoveryProject = {
 
       //Check if the URL is know to be bad: private, about:, odd ports, etc.
       if (
-        !sanitizeUrl(activeURL, { testMode: WebDiscoveryProject.testMode })
-          .safeUrl
+        sanitizeUrl(activeURL, {
+          testMode: WebDiscoveryProject.testMode,
+        }).result !== "safe"
       ) {
         logger.debug("[onLocationChange] isSuspiciousURL", activeURL);
         return;
@@ -2329,28 +2335,22 @@ const WebDiscoveryProject = {
           }
         }
 
+        let sanitizedUrl = sanitizeUrl(linkURL, {
+          testMode: WebDiscoveryProject.testMode,
+        });
         if (
-          sanitizeUrl(linkURL, { testMode: WebDiscoveryProject.testMode })
-            .safeUrl &&
+          sanitizedUrl.result === "safe" &&
           !WebDiscoveryProject.dropLongURL(linkURL)
         ) {
           WebDiscoveryProject.isAlreadyMarkedPrivate(linkURL, function (_res) {
             if (_res && _res["private"] == 0) {
               WebDiscoveryProject.state["v"][activeURL]["c"].push({
-                l:
-                  "" +
-                  sanitizeUrl(linkURL, {
-                    testMode: WebDiscoveryProject.testMode,
-                  }).safeUrl,
+                l: "" + sanitizedUrl.safeUrl,
                 t: WebDiscoveryProject.counter,
               });
             } else if (!_res) {
               WebDiscoveryProject.state["v"][activeURL]["c"].push({
-                l:
-                  "" +
-                  sanitizeUrl(linkURL, {
-                    testMode: WebDiscoveryProject.testMode,
-                  }).safeUrl,
+                l: "" + sanitizedUrl.safeUrl,
                 t: WebDiscoveryProject.counter,
               });
             }
@@ -2436,7 +2436,7 @@ const WebDiscoveryProject = {
   init: function () {
     return Promise.resolve().then(() => {
       logger.debug("Init function called:");
-      WebDiscoveryProject.logger = logger
+      WebDiscoveryProject.logger = logger;
       return Promise.resolve()
         .then(() => {
           if (WebDiscoveryProject.db) {
@@ -2570,18 +2570,9 @@ const WebDiscoveryProject = {
       // Check if they are suspicious.
       // Check if they are marked private.
       if (msg.payload.ref) {
-        if (
-          !sanitizeUrl(msg.payload["ref"], {
-            testMode: WebDiscoveryProject.testMode,
-          }).safeUrl
-        ) {
-          msg.payload["ref"] = null;
-        } else {
-          msg.payload["ref"] = sanitizeUrl(
-            msg.payload["ref"],
-            { testMode: WebDiscoveryProject.testMode },
-          ).safeUrl;
-        }
+        msg.payload["ref"] = sanitizeUrl(msg.payload["ref"], {
+          testMode: WebDiscoveryProject.testMode,
+        }).safeUrl;
 
         // Check if ref. exists in bloom filter, then turn ref to null.
         WebDiscoveryProject.isAlreadyMarkedPrivate(
@@ -2658,9 +2649,9 @@ const WebDiscoveryProject = {
 
       // check if suspiciousURL
       if (
-        !sanitizeUrl(msg.payload.url, {
+        sanitizeUrl(msg.payload.url, {
           testMode: WebDiscoveryProject.testMode,
-        }).safeUrl
+        }).result !== "safe"
       )
         return null;
 
@@ -2669,9 +2660,9 @@ const WebDiscoveryProject = {
         msg.payload.x.canonical_url != ""
       ) {
         if (
-          !sanitizeUrl(msg.payload.x.canonical_url, {
+          sanitizeUrl(msg.payload.x.canonical_url, {
             testMode: WebDiscoveryProject.testMode,
-          }).safeUrl
+          }).result !== "safe"
         )
           return null;
       }
@@ -2680,11 +2671,11 @@ const WebDiscoveryProject = {
       if (msg.payload.red) {
         var cleanRed = [];
         msg.payload.red.forEach(function (e) {
-          if (sanitizeUrl(e, { testMode: WebDiscoveryProject.testMode }).result !== "safe") {
-            cleanRed.push(
-              sanitizeUrl(e, { testMode: WebDiscoveryProject.testMode })
-                .safeUrl,
-            );
+          let safeUrl = sanitizeUrl(e, {
+            testMode: WebDiscoveryProject.testMode,
+          }).safeUrl;
+          if (safeUrl) {
+            cleanRed.push(safeUrl);
           }
         });
         msg.payload.red = cleanRed;
@@ -2772,7 +2763,7 @@ const WebDiscoveryProject = {
           if (
             sanitizeUrl(msg.payload.r[eachResult].u, {
               testMode: WebDiscoveryProject.testMode,
-            }).safeUrl
+            }).result === "safe"
           ) {
             cleanR.push(msg.payload.r[eachResult]);
           }
@@ -2986,7 +2977,10 @@ const WebDiscoveryProject = {
     if (!source || source === "openTabs") {
       const allOpenPages = await WebDiscoveryProject.getAllOpenPages();
       pages = allOpenPages
-        .map((url) => ({ url, page_doc: WebDiscoveryProject.state.v[url] }))
+        .map((url) => ({
+          url,
+          page_doc: WebDiscoveryProject.state.v[url],
+        }))
         .filter(({ url, page_doc }) => page_doc && isRelevantUrl(url));
     } else if (source === "unprocessed") {
       pages = await new Promise((resolve, reject) => {
@@ -3458,8 +3452,8 @@ const WebDiscoveryProject = {
 
     if (
       queryLikeURL &&
-      (!sanitizeUrl(query, { testMode: WebDiscoveryProject.testMode })
-        .safeUrl ||
+      (sanitizeUrl(query, { testMode: WebDiscoveryProject.testMode }).result !==
+        "safe" ||
         WebDiscoveryProject.dropLongURL(query))
     ) {
       logger.debug("Query is dangerous");
@@ -3483,7 +3477,8 @@ const WebDiscoveryProject = {
 
       // Check URL is suspicious
       if (
-        sanitizeUrl(url, { testMode: WebDiscoveryProject.testMode }).result !== "safe"
+        sanitizeUrl(url, { testMode: WebDiscoveryProject.testMode }).result !==
+        "safe"
       ) {
         logger.debug("Url is suspicious");
         url = "(PROTECTED)";
@@ -3856,7 +3851,8 @@ const WebDiscoveryProject = {
           setPrivate = true;
           logger.debug("Setting private because empty page data");
         } else if (
-          sanitizeUrl(url, { testMode: WebDiscoveryProject.testMode }).result !== "safe"
+          sanitizeUrl(url, { testMode: WebDiscoveryProject.testMode })
+            .result !== "safe"
         ) {
           // if the url looks private already add it already as checked and private
           let reason = "susp. url";