[PS-1123] Improve hostname and domain retrieval (#3168)

* Add test cases from previous PR https://github.com/bitwarden/jslib/pull/547

* Install tldts as replacement for tldjs

* Use tldts for hostname and domain retrieval/validation

* Remove usage of old tldjs.noop-implementation

* Add handling of about protocol

* Remove usage of tldEndingRegex and use tldts check instead

* Uninstall @types/tldjs and tldjs

* Updated package-lock.json

* Fix accessibility cookie check

* Rename loginUriView.spec to login-uri-view.spec

* Add test for getDomain failing file links

* getHostName - Return null when given, data, about or file links

libs/common/spec/misc/utils.spec.ts

@@ -14,27 +14,105 @@ describe("Utils Service", () => {
       expect(Utils.getDomain("data:image/jpeg;base64,AAA")).toBeNull();
     });
 
+    it("should fail for about urls", () => {
+      expect(Utils.getDomain("about")).toBeNull();
+      expect(Utils.getDomain("about:")).toBeNull();
+      expect(Utils.getDomain("about:blank")).toBeNull();
+    });
+
+    it("should fail for file url", () => {
+      expect(Utils.getDomain("file:///C://somefolder/form.pdf")).toBeNull();
+    });
+
     it("should handle urls without protocol", () => {
       expect(Utils.getDomain("bitwarden.com")).toBe("bitwarden.com");
       expect(Utils.getDomain("wrong://bitwarden.com")).toBe("bitwarden.com");
     });
 
     it("should handle valid urls", () => {
-      expect(Utils.getDomain("https://bitwarden")).toBe("bitwarden");
-      expect(Utils.getDomain("https://bitwarden.com")).toBe("bitwarden.com");
+      expect(Utils.getDomain("bitwarden.com")).toBe("bitwarden.com");
       expect(Utils.getDomain("http://bitwarden.com")).toBe("bitwarden.com");
+      expect(Utils.getDomain("https://bitwarden.com")).toBe("bitwarden.com");
+
+      expect(Utils.getDomain("www.bitwarden.com")).toBe("bitwarden.com");
+      expect(Utils.getDomain("http://www.bitwarden.com")).toBe("bitwarden.com");
+      expect(Utils.getDomain("https://www.bitwarden.com")).toBe("bitwarden.com");
+
+      expect(Utils.getDomain("vault.bitwarden.com")).toBe("bitwarden.com");
       expect(Utils.getDomain("http://vault.bitwarden.com")).toBe("bitwarden.com");
+      expect(Utils.getDomain("https://vault.bitwarden.com")).toBe("bitwarden.com");
+
+      expect(Utils.getDomain("www.vault.bitwarden.com")).toBe("bitwarden.com");
+      expect(Utils.getDomain("http://www.vault.bitwarden.com")).toBe("bitwarden.com");
+      expect(Utils.getDomain("https://www.vault.bitwarden.com")).toBe("bitwarden.com");
+
+      expect(
+        Utils.getDomain("user:password@bitwarden.com:8080/password/sites?and&query#hash")
+      ).toBe("bitwarden.com");
+      expect(
+        Utils.getDomain("http://user:password@bitwarden.com:8080/password/sites?and&query#hash")
+      ).toBe("bitwarden.com");
       expect(
         Utils.getDomain("https://user:password@bitwarden.com:8080/password/sites?and&query#hash")
       ).toBe("bitwarden.com");
+
+      expect(Utils.getDomain("bitwarden.unknown")).toBe("bitwarden.unknown");
+      expect(Utils.getDomain("http://bitwarden.unknown")).toBe("bitwarden.unknown");
       expect(Utils.getDomain("https://bitwarden.unknown")).toBe("bitwarden.unknown");
     });
 
-    it("should support localhost and IP", () => {
+    it("should handle valid urls with an underscore in subdomain", () => {
+      expect(Utils.getDomain("my_vault.bitwarden.com/")).toBe("bitwarden.com");
+      expect(Utils.getDomain("http://my_vault.bitwarden.com/")).toBe("bitwarden.com");
+      expect(Utils.getDomain("https://my_vault.bitwarden.com/")).toBe("bitwarden.com");
+    });
+
+    it("should support urls containing umlauts", () => {
+      expect(Utils.getDomain("bütwarden.com")).toBe("bütwarden.com");
+      expect(Utils.getDomain("http://bütwarden.com")).toBe("bütwarden.com");
+      expect(Utils.getDomain("https://bütwarden.com")).toBe("bütwarden.com");
+
+      expect(Utils.getDomain("subdomain.bütwarden.com")).toBe("bütwarden.com");
+      expect(Utils.getDomain("http://subdomain.bütwarden.com")).toBe("bütwarden.com");
+      expect(Utils.getDomain("https://subdomain.bütwarden.com")).toBe("bütwarden.com");
+    });
+
+    it("should support punycode urls", () => {
+      expect(Utils.getDomain("xn--btwarden-65a.com")).toBe("xn--btwarden-65a.com");
+      expect(Utils.getDomain("xn--btwarden-65a.com")).toBe("xn--btwarden-65a.com");
+      expect(Utils.getDomain("xn--btwarden-65a.com")).toBe("xn--btwarden-65a.com");
+
+      expect(Utils.getDomain("subdomain.xn--btwarden-65a.com")).toBe("xn--btwarden-65a.com");
+      expect(Utils.getDomain("http://subdomain.xn--btwarden-65a.com")).toBe("xn--btwarden-65a.com");
+      expect(Utils.getDomain("https://subdomain.xn--btwarden-65a.com")).toBe(
+        "xn--btwarden-65a.com"
+      );
+    });
+
+    it("should support localhost", () => {
+      expect(Utils.getDomain("localhost")).toBe("localhost");
+      expect(Utils.getDomain("http://localhost")).toBe("localhost");
       expect(Utils.getDomain("https://localhost")).toBe("localhost");
+    });
+
+    it("should support localhost with subdomain", () => {
+      expect(Utils.getDomain("subdomain.localhost")).toBe("localhost");
+      expect(Utils.getDomain("http://subdomain.localhost")).toBe("localhost");
+      expect(Utils.getDomain("https://subdomain.localhost")).toBe("localhost");
+    });
+
+    it("should support IPv4", () => {
+      expect(Utils.getDomain("192.168.1.1")).toBe("192.168.1.1");
+      expect(Utils.getDomain("http://192.168.1.1")).toBe("192.168.1.1");
       expect(Utils.getDomain("https://192.168.1.1")).toBe("192.168.1.1");
     });
 
+    it("should support IPv6", () => {
+      expect(Utils.getDomain("[2620:fe::fe]")).toBe("2620:fe::fe");
+      expect(Utils.getDomain("http://[2620:fe::fe]")).toBe("2620:fe::fe");
+      expect(Utils.getDomain("https://[2620:fe::fe]")).toBe("2620:fe::fe");
+    });
+
     it("should reject invalid hostnames", () => {
       expect(Utils.getDomain("https://mywebsite.com$.mywebsite.com")).toBeNull();
       expect(Utils.getDomain("https://mywebsite.com!.mywebsite.com")).toBeNull();
@@ -47,20 +125,107 @@ describe("Utils Service", () => {
       expect(Utils.getHostname(undefined)).toBeNull();
       expect(Utils.getHostname(" ")).toBeNull();
       expect(Utils.getHostname('https://bit!:"_&ward.com')).toBeNull();
-      expect(Utils.getHostname("bitwarden")).toBeNull();
+    });
+
+    it("should fail for data urls", () => {
+      expect(Utils.getHostname("data:image/jpeg;base64,AAA")).toBeNull();
+    });
+
+    it("should fail for about urls", () => {
+      expect(Utils.getHostname("about")).toBe("about");
+      expect(Utils.getHostname("about:")).toBeNull();
+      expect(Utils.getHostname("about:blank")).toBeNull();
+    });
+
+    it("should fail for file url", () => {
+      expect(Utils.getHostname("file:///C:/somefolder/form.pdf")).toBeNull();
     });
 
     it("should handle valid urls", () => {
+      expect(Utils.getHostname("bitwarden")).toBe("bitwarden");
+      expect(Utils.getHostname("http://bitwarden")).toBe("bitwarden");
+      expect(Utils.getHostname("https://bitwarden")).toBe("bitwarden");
+
       expect(Utils.getHostname("bitwarden.com")).toBe("bitwarden.com");
-      expect(Utils.getHostname("https://bitwarden.com")).toBe("bitwarden.com");
       expect(Utils.getHostname("http://bitwarden.com")).toBe("bitwarden.com");
+      expect(Utils.getHostname("https://bitwarden.com")).toBe("bitwarden.com");
+
+      expect(Utils.getHostname("www.bitwarden.com")).toBe("www.bitwarden.com");
+      expect(Utils.getHostname("http://www.bitwarden.com")).toBe("www.bitwarden.com");
+      expect(Utils.getHostname("https://www.bitwarden.com")).toBe("www.bitwarden.com");
+
+      expect(Utils.getHostname("vault.bitwarden.com")).toBe("vault.bitwarden.com");
       expect(Utils.getHostname("http://vault.bitwarden.com")).toBe("vault.bitwarden.com");
+      expect(Utils.getHostname("https://vault.bitwarden.com")).toBe("vault.bitwarden.com");
+
+      expect(Utils.getHostname("www.vault.bitwarden.com")).toBe("www.vault.bitwarden.com");
+      expect(Utils.getHostname("http://www.vault.bitwarden.com")).toBe("www.vault.bitwarden.com");
+      expect(Utils.getHostname("https://www.vault.bitwarden.com")).toBe("www.vault.bitwarden.com");
+
+      expect(
+        Utils.getHostname("user:password@bitwarden.com:8080/password/sites?and&query#hash")
+      ).toBe("bitwarden.com");
+      expect(
+        Utils.getHostname("https://user:password@bitwarden.com:8080/password/sites?and&query#hash")
+      ).toBe("bitwarden.com");
+      expect(Utils.getHostname("https://bitwarden.unknown")).toBe("bitwarden.unknown");
     });
 
-    it("should support localhost and IP", () => {
+    it("should handle valid urls with an underscore in subdomain", () => {
+      expect(Utils.getHostname("my_vault.bitwarden.com/")).toBe("my_vault.bitwarden.com");
+      expect(Utils.getHostname("http://my_vault.bitwarden.com/")).toBe("my_vault.bitwarden.com");
+      expect(Utils.getHostname("https://my_vault.bitwarden.com/")).toBe("my_vault.bitwarden.com");
+    });
+
+    it("should support urls containing umlauts", () => {
+      expect(Utils.getHostname("bütwarden.com")).toBe("bütwarden.com");
+      expect(Utils.getHostname("http://bütwarden.com")).toBe("bütwarden.com");
+      expect(Utils.getHostname("https://bütwarden.com")).toBe("bütwarden.com");
+
+      expect(Utils.getHostname("subdomain.bütwarden.com")).toBe("subdomain.bütwarden.com");
+      expect(Utils.getHostname("http://subdomain.bütwarden.com")).toBe("subdomain.bütwarden.com");
+      expect(Utils.getHostname("https://subdomain.bütwarden.com")).toBe("subdomain.bütwarden.com");
+    });
+
+    it("should support punycode urls", () => {
+      expect(Utils.getHostname("xn--btwarden-65a.com")).toBe("xn--btwarden-65a.com");
+      expect(Utils.getHostname("xn--btwarden-65a.com")).toBe("xn--btwarden-65a.com");
+      expect(Utils.getHostname("xn--btwarden-65a.com")).toBe("xn--btwarden-65a.com");
+
+      expect(Utils.getHostname("subdomain.xn--btwarden-65a.com")).toBe(
+        "subdomain.xn--btwarden-65a.com"
+      );
+      expect(Utils.getHostname("http://subdomain.xn--btwarden-65a.com")).toBe(
+        "subdomain.xn--btwarden-65a.com"
+      );
+      expect(Utils.getHostname("https://subdomain.xn--btwarden-65a.com")).toBe(
+        "subdomain.xn--btwarden-65a.com"
+      );
+    });
+
+    it("should support localhost", () => {
+      expect(Utils.getHostname("localhost")).toBe("localhost");
+      expect(Utils.getHostname("http://localhost")).toBe("localhost");
       expect(Utils.getHostname("https://localhost")).toBe("localhost");
+    });
+
+    it("should support localhost with subdomain", () => {
+      expect(Utils.getHostname("subdomain.localhost")).toBe("subdomain.localhost");
+      expect(Utils.getHostname("http://subdomain.localhost")).toBe("subdomain.localhost");
+      expect(Utils.getHostname("https://subdomain.localhost")).toBe("subdomain.localhost");
+    });
+
+    it("should support IPv4", () => {
+      expect(Utils.getHostname("192.168.1.1")).toBe("192.168.1.1");
+      expect(Utils.getHostname("http://192.168.1.1")).toBe("192.168.1.1");
       expect(Utils.getHostname("https://192.168.1.1")).toBe("192.168.1.1");
     });
+
+    it("should support IPv6", () => {
+      expect(Utils.getHostname("[2620:fe::fe]")).toBe("2620:fe::fe");
+      expect(Utils.getHostname("http://[2620:fe::fe]")).toBe("2620:fe::fe");
+      expect(Utils.getHostname("https://[2620:fe::fe]")).toBe("2620:fe::fe");
+    });
   });
 
   describe("newGuid", () => {

libs/common/spec/view/login-uri-view.spec.ts

@@ -0,0 +1,65 @@
+import { UriMatchType } from "@bitwarden/common/enums/uriMatchType";
+import { LoginUriView } from "@bitwarden/common/models/view/login-uri.view";
+
+const testData = [
+  {
+    match: UriMatchType.Host,
+    uri: "http://example.com/login",
+    expected: "http://example.com/login",
+  },
+  {
+    match: UriMatchType.Host,
+    uri: "bitwarden.com",
+    expected: "http://bitwarden.com",
+  },
+  {
+    match: UriMatchType.Host,
+    uri: "bitwarden.de",
+    expected: "http://bitwarden.de",
+  },
+  {
+    match: UriMatchType.Host,
+    uri: "bitwarden.br",
+    expected: "http://bitwarden.br",
+  },
+];
+
+describe("LoginUriView", () => {
+  it("isWebsite() given an invalid domain should return false", async () => {
+    const uri = new LoginUriView();
+    Object.assign(uri, { match: UriMatchType.Host, uri: "bit!:_&ward.com" });
+    expect(uri.isWebsite).toBe(false);
+  });
+
+  testData.forEach((data) => {
+    it(`isWebsite() given ${data.uri} should return true`, async () => {
+      const uri = new LoginUriView();
+      Object.assign(uri, { match: data.match, uri: data.uri });
+      expect(uri.isWebsite).toBe(true);
+    });
+
+    it(`launchUri() given ${data.uri} should return ${data.expected}`, async () => {
+      const uri = new LoginUriView();
+      Object.assign(uri, { match: data.match, uri: data.uri });
+      expect(uri.launchUri).toBe(data.expected);
+    });
+
+    it(`canLaunch() given ${data.uri} should return true`, async () => {
+      const uri = new LoginUriView();
+      Object.assign(uri, { match: data.match, uri: data.uri });
+      expect(uri.canLaunch).toBe(true);
+    });
+  });
+
+  it(`canLaunch should return false when MatchDetection is set to Regex`, async () => {
+    const uri = new LoginUriView();
+    Object.assign(uri, { match: UriMatchType.RegularExpression, uri: "bitwarden.com" });
+    expect(uri.canLaunch).toBe(false);
+  });
+
+  it(`canLaunch() should return false when the given protocol does not match CanLaunchWhiteList`, async () => {
+    const uri = new LoginUriView();
+    Object.assign(uri, { match: UriMatchType.Host, uri: "someprotocol://bitwarden.com" });
+    expect(uri.canLaunch).toBe(false);
+  });
+});

libs/common/src/misc/tldjs.noop.ts

@@ -1,7 +0,0 @@
-export function getDomain(host: string): string | null {
-  return null;
-}
-
-export function isValid(host: string): boolean {
-  return true;
-}

libs/common/src/misc/utils.ts

@@ -1,5 +1,5 @@
 /* eslint-disable no-useless-escape */
-import * as tldjs from "tldjs";
+import { getHostname, parse } from "tldts";
 
 import { AbstractEncryptService } from "../abstractions/abstractEncrypt.service";
 import { CryptoService } from "../abstractions/crypto.service";
@@ -24,11 +24,10 @@ export class Utils {
   static isMobileBrowser = false;
   static isAppleMobileBrowser = false;
   static global: typeof global = null;
-  static tldEndingRegex =
-    /.*\.(com|net|org|edu|uk|gov|ca|de|jp|fr|au|ru|ch|io|es|us|co|xyz|info|ly|mil)$/;
   // Transpiled version of /\p{Emoji_Presentation}/gu using https://mothereff.in/regexpu. Used for compatability in older browsers.
   static regexpEmojiPresentation =
     /(?:[\u231A\u231B\u23E9-\u23EC\u23F0\u23F3\u25FD\u25FE\u2614\u2615\u2648-\u2653\u267F\u2693\u26A1\u26AA\u26AB\u26BD\u26BE\u26C4\u26C5\u26CE\u26D4\u26EA\u26F2\u26F3\u26F5\u26FA\u26FD\u2705\u270A\u270B\u2728\u274C\u274E\u2753-\u2755\u2757\u2795-\u2797\u27B0\u27BF\u2B1B\u2B1C\u2B50\u2B55]|\uD83C[\uDC04\uDCCF\uDD8E\uDD91-\uDD9A\uDDE6-\uDDFF\uDE01\uDE1A\uDE2F\uDE32-\uDE36\uDE38-\uDE3A\uDE50\uDE51\uDF00-\uDF20\uDF2D-\uDF35\uDF37-\uDF7C\uDF7E-\uDF93\uDFA0-\uDFCA\uDFCF-\uDFD3\uDFE0-\uDFF0\uDFF4\uDFF8-\uDFFF]|\uD83D[\uDC00-\uDC3E\uDC40\uDC42-\uDCFC\uDCFF-\uDD3D\uDD4B-\uDD4E\uDD50-\uDD67\uDD7A\uDD95\uDD96\uDDA4\uDDFB-\uDE4F\uDE80-\uDEC5\uDECC\uDED0-\uDED2\uDED5-\uDED7\uDEEB\uDEEC\uDEF4-\uDEFC\uDFE0-\uDFEB]|\uD83E[\uDD0C-\uDD3A\uDD3C-\uDD45\uDD47-\uDD78\uDD7A-\uDDCB\uDDCD-\uDDFF\uDE70-\uDE74\uDE78-\uDE7A\uDE80-\uDE86\uDE90-\uDEA8\uDEB0-\uDEB6\uDEC0-\uDEC2\uDED0-\uDED6])/g;
+  static readonly validHosts: string[] = ["localhost"];
 
   static init() {
     if (Utils.inited) {
@@ -214,12 +213,39 @@ export class Utils {
   }
 
   static getHostname(uriString: string): string {
-    const url = Utils.getUrl(uriString);
+    if (Utils.isNullOrWhitespace(uriString)) {
+      return null;
+    }
+
+    uriString = uriString.trim();
+
+    if (uriString.startsWith("data:")) {
+      return null;
+    }
+
+    if (uriString.startsWith("about:")) {
+      return null;
+    }
+
+    if (uriString.startsWith("file:")) {
+      return null;
+    }
+
+    // Does uriString contain invalid characters
+    // TODO Needs to possibly be extended, although '!' is a reserved character
+    if (uriString.indexOf("!") > 0) {
+      return null;
+    }
+
     try {
-      return url != null && url.hostname !== "" ? url.hostname : null;
+      const hostname = getHostname(uriString, { validHosts: this.validHosts });
+      if (hostname != null) {
+        return hostname;
+      }
     } catch {
       return null;
     }
+    return null;
   }
 
   static getHost(uriString: string): string {
@@ -232,60 +258,35 @@ export class Utils {
   }
 
   static getDomain(uriString: string): string {
-    if (uriString == null) {
+    if (Utils.isNullOrWhitespace(uriString)) {
       return null;
     }
 
     uriString = uriString.trim();
-    if (uriString === "") {
-      return null;
-    }
 
     if (uriString.startsWith("data:")) {
       return null;
     }
 
-    let httpUrl = uriString.startsWith("http://") || uriString.startsWith("https://");
-    if (
-      !httpUrl &&
-      uriString.indexOf("://") < 0 &&
-      Utils.tldEndingRegex.test(uriString) &&
-      uriString.indexOf("@") < 0
-    ) {
-      uriString = "http://" + uriString;
-      httpUrl = true;
-    }
-
-    if (httpUrl) {
-      try {
-        const url = Utils.getUrlObject(uriString);
-        const validHostname = tldjs?.isValid != null ? tldjs.isValid(url.hostname) : true;
-        if (!validHostname) {
-          return null;
-        }
-
-        if (url.hostname === "localhost" || Utils.validIpAddress(url.hostname)) {
-          return url.hostname;
-        }
-
-        const urlDomain =
-          tldjs != null && tldjs.getDomain != null ? tldjs.getDomain(url.hostname) : null;
-        return urlDomain != null ? urlDomain : url.hostname;
-      } catch (e) {
-        // Invalid domain, try another approach below.
-      }
+    if (uriString.startsWith("about:")) {
+      return null;
     }
 
     try {
-      const domain = tldjs != null && tldjs.getDomain != null ? tldjs.getDomain(uriString) : null;
+      const parseResult = parse(uriString, { validHosts: this.validHosts });
+      if (parseResult != null && parseResult.hostname != null) {
+        if (parseResult.hostname === "localhost" || parseResult.isIp) {
+          return parseResult.hostname;
+        }
 
-      if (domain != null) {
-        return domain;
+        if (parseResult.domain != null) {
+          return parseResult.domain;
+        }
+        return null;
       }
     } catch {
       return null;
     }
-
     return null;
   }
 
@@ -358,14 +359,11 @@ export class Utils {
   }
 
   static getUrl(uriString: string): URL {
-    if (uriString == null) {
+    if (this.isNullOrWhitespace(uriString)) {
       return null;
     }
 
     uriString = uriString.trim();
-    if (uriString === "") {
-      return null;
-    }
 
     let url = Utils.getUrlObject(uriString);
     if (url == null) {
@@ -425,12 +423,6 @@ export class Utils {
     return this.global.bitwardenContainerService;
   }
 
-  private static validIpAddress(ipString: string): boolean {
-    const ipRegex =
-      /^(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)$/;
-    return ipRegex.test(ipString);
-  }
-
   private static isMobile(win: Window) {
     let mobile = false;
     ((a) => {

libs/common/src/models/view/login-uri.view.ts

@@ -100,7 +100,7 @@ export class LoginUriView implements View {
       this.uri != null &&
       (this.uri.indexOf("http://") === 0 ||
         this.uri.indexOf("https://") === 0 ||
-        (this.uri.indexOf("://") < 0 && Utils.tldEndingRegex.test(this.uri)))
+        (this.uri.indexOf("://") < 0 && !Utils.isNullOrWhitespace(Utils.getDomain(this.uri))))
     );
   }
 
@@ -122,7 +122,7 @@ export class LoginUriView implements View {
   }
 
   get launchUri(): string {
-    return this.uri.indexOf("://") < 0 && Utils.tldEndingRegex.test(this.uri)
+    return this.uri.indexOf("://") < 0 && !Utils.isNullOrWhitespace(Utils.getDomain(this.uri))
       ? "http://" + this.uri
       : this.uri;
   }