Added parsing of urls with tldts with fallback to tldjs

common/package-lock.json

@@ -18,6 +18,7 @@
         "papaparse": "^5.3.0",
         "rxjs": "6.6.7",
         "tldjs": "^2.3.1",
+        "tldts": "5.7.52",
         "zxcvbn": "^4.4.2"
       },
       "devDependencies": {
@@ -430,6 +431,22 @@
         "node": ">= 4"
       }
     },
+    "node_modules/tldts": {
+      "version": "5.7.52",
+      "resolved": "https://registry.npmjs.org/tldts/-/tldts-5.7.52.tgz",
+      "integrity": "sha512-0ddIYX/04P8qFVY0XYAlF7vJEef+nd+natGSnlasB8MRC4iZIz5HzX4G0Vm6jRItxEtWUuXo2+eeBLBtsZZgEQ==",
+      "dependencies": {
+        "tldts-core": "^5.7.52"
+      },
+      "bin": {
+        "tldts": "bin/cli.js"
+      }
+    },
+    "node_modules/tldts-core": {
+      "version": "5.7.52",
+      "resolved": "https://registry.npmjs.org/tldts-core/-/tldts-core-5.7.52.tgz",
+      "integrity": "sha512-xLa9sSdmSIec2kok76Fv68vvf58QZJFGEOan/pLsk9rrPfkfMZq7Vr03X0hPr8VCc09tcbfU6izXzRDl8oItYQ=="
+    },
     "node_modules/tough-cookie": {
       "version": "2.5.0",
       "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.5.0.tgz",
@@ -868,6 +885,19 @@
         "punycode": "^1.4.1"
       }
     },
+    "tldts": {
+      "version": "5.7.52",
+      "resolved": "https://registry.npmjs.org/tldts/-/tldts-5.7.52.tgz",
+      "integrity": "sha512-0ddIYX/04P8qFVY0XYAlF7vJEef+nd+natGSnlasB8MRC4iZIz5HzX4G0Vm6jRItxEtWUuXo2+eeBLBtsZZgEQ==",
+      "requires": {
+        "tldts-core": "^5.7.52"
+      }
+    },
+    "tldts-core": {
+      "version": "5.7.52",
+      "resolved": "https://registry.npmjs.org/tldts-core/-/tldts-core-5.7.52.tgz",
+      "integrity": "sha512-xLa9sSdmSIec2kok76Fv68vvf58QZJFGEOan/pLsk9rrPfkfMZq7Vr03X0hPr8VCc09tcbfU6izXzRDl8oItYQ=="
+    },
     "tough-cookie": {
       "version": "2.5.0",
       "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.5.0.tgz",

common/package.json

@@ -39,6 +39,7 @@
     "papaparse": "^5.3.0",
     "rxjs": "6.6.7",
     "tldjs": "^2.3.1",
+    "tldts": "5.7.52",
     "zxcvbn": "^4.4.2"
   }
 }

common/src/misc/utils.ts

@@ -1,4 +1,5 @@
 import * as tldjs from 'tldjs';
+import * as tldts from 'tldts';
 
 import { I18nService } from '../abstractions/i18n.service';
 
@@ -15,7 +16,8 @@ export class Utils {
     static global: any = null;
     static tldEndingRegex = /.*\.(com|net|org|edu|uk|gov|ca|de|jp|fr|au|ru|ch|io|es|us|co|xyz|info|ly|mil)$/;
     // Transpiled version of /\p{Emoji_Presentation}/gu using https://mothereff.in/regexpu. Used for compatability in older browsers.
-    static regexpEmojiPresentation =  /(?:[\u231A\u231B\u23E9-\u23EC\u23F0\u23F3\u25FD\u25FE\u2614\u2615\u2648-\u2653\u267F\u2693\u26A1\u26AA\u26AB\u26BD\u26BE\u26C4\u26C5\u26CE\u26D4\u26EA\u26F2\u26F3\u26F5\u26FA\u26FD\u2705\u270A\u270B\u2728\u274C\u274E\u2753-\u2755\u2757\u2795-\u2797\u27B0\u27BF\u2B1B\u2B1C\u2B50\u2B55]|\uD83C[\uDC04\uDCCF\uDD8E\uDD91-\uDD9A\uDDE6-\uDDFF\uDE01\uDE1A\uDE2F\uDE32-\uDE36\uDE38-\uDE3A\uDE50\uDE51\uDF00-\uDF20\uDF2D-\uDF35\uDF37-\uDF7C\uDF7E-\uDF93\uDFA0-\uDFCA\uDFCF-\uDFD3\uDFE0-\uDFF0\uDFF4\uDFF8-\uDFFF]|\uD83D[\uDC00-\uDC3E\uDC40\uDC42-\uDCFC\uDCFF-\uDD3D\uDD4B-\uDD4E\uDD50-\uDD67\uDD7A\uDD95\uDD96\uDDA4\uDDFB-\uDE4F\uDE80-\uDEC5\uDECC\uDED0-\uDED2\uDED5-\uDED7\uDEEB\uDEEC\uDEF4-\uDEFC\uDFE0-\uDFEB]|\uD83E[\uDD0C-\uDD3A\uDD3C-\uDD45\uDD47-\uDD78\uDD7A-\uDDCB\uDDCD-\uDDFF\uDE70-\uDE74\uDE78-\uDE7A\uDE80-\uDE86\uDE90-\uDEA8\uDEB0-\uDEB6\uDEC0-\uDEC2\uDED0-\uDED6])/g;
+    static regexpEmojiPresentation = /(?:[\u231A\u231B\u23E9-\u23EC\u23F0\u23F3\u25FD\u25FE\u2614\u2615\u2648-\u2653\u267F\u2693\u26A1\u26AA\u26AB\u26BD\u26BE\u26C4\u26C5\u26CE\u26D4\u26EA\u26F2\u26F3\u26F5\u26FA\u26FD\u2705\u270A\u270B\u2728\u274C\u274E\u2753-\u2755\u2757\u2795-\u2797\u27B0\u27BF\u2B1B\u2B1C\u2B50\u2B55]|\uD83C[\uDC04\uDCCF\uDD8E\uDD91-\uDD9A\uDDE6-\uDDFF\uDE01\uDE1A\uDE2F\uDE32-\uDE36\uDE38-\uDE3A\uDE50\uDE51\uDF00-\uDF20\uDF2D-\uDF35\uDF37-\uDF7C\uDF7E-\uDF93\uDFA0-\uDFCA\uDFCF-\uDFD3\uDFE0-\uDFF0\uDFF4\uDFF8-\uDFFF]|\uD83D[\uDC00-\uDC3E\uDC40\uDC42-\uDCFC\uDCFF-\uDD3D\uDD4B-\uDD4E\uDD50-\uDD67\uDD7A\uDD95\uDD96\uDDA4\uDDFB-\uDE4F\uDE80-\uDEC5\uDECC\uDED0-\uDED2\uDED5-\uDED7\uDEEB\uDEEC\uDEF4-\uDEFC\uDFE0-\uDFEB]|\uD83E[\uDD0C-\uDD3A\uDD3C-\uDD45\uDD47-\uDD78\uDD7A-\uDDCB\uDDCD-\uDDFF\uDE70-\uDE74\uDE78-\uDE7A\uDE80-\uDE86\uDE90-\uDEA8\uDEB0-\uDEB6\uDEC0-\uDEC2\uDED0-\uDED6])/g;
+    static readonly validHosts: string[] = ['localhost'];
 
     static init() {
         if (Utils.inited) {
@@ -187,6 +189,28 @@ export class Utils {
     }
 
     static getHostname(uriString: string): string {
+        if (uriString == null) {
+            return null;
+        }
+
+        uriString = uriString.trim();
+        if (uriString === '') {
+            return null;
+        }
+
+        if (tldts.getHostname != null) {
+            // Does uriString contain invalid characters
+            // TODO Needs to possibly be extended, although '!' is a reserved character
+            if (uriString.indexOf('!') > 0) {
+                return null;
+            }
+
+            const hostname = tldts.getHostname(uriString, { validHosts: this.validHosts });
+            if (hostname != null) {
+                return hostname;
+            }
+        }
+
         const url = Utils.getUrl(uriString);
         try {
             return url != null && url.hostname !== '' ? url.hostname : null;
@@ -225,6 +249,18 @@ export class Utils {
             httpUrl = true;
         }
 
+        const parseResult = tldts.parse != null ? tldts.parse(uriString, { validHosts: this.validHosts }) : null;
+        if (parseResult != null && parseResult.hostname != null) {
+            if (parseResult.hostname === 'localhost' || parseResult.isIp) {
+                return parseResult.hostname;
+            }
+
+            if (parseResult.domain != null) {
+                return parseResult.domain;
+            }
+            return null;
+        }
+
         if (httpUrl) {
             try {
                 const url = Utils.getUrlObject(uriString);

package-lock.json

@@ -87,6 +87,7 @@
         "papaparse": "^5.3.0",
         "rxjs": "6.6.7",
         "tldjs": "^2.3.1",
+        "tldts": "5.7.52",
         "zxcvbn": "^4.4.2"
       },
       "devDependencies": {
@@ -7641,6 +7642,22 @@
       "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.4.1.tgz",
       "integrity": "sha1-wNWmOycYgArY4esPpSachN1BhF4="
     },
+    "node_modules/tldts": {
+      "version": "5.7.52",
+      "resolved": "https://registry.npmjs.org/tldts/-/tldts-5.7.52.tgz",
+      "integrity": "sha512-0ddIYX/04P8qFVY0XYAlF7vJEef+nd+natGSnlasB8MRC4iZIz5HzX4G0Vm6jRItxEtWUuXo2+eeBLBtsZZgEQ==",
+      "dependencies": {
+        "tldts-core": "^5.7.52"
+      },
+      "bin": {
+        "tldts": "bin/cli.js"
+      }
+    },
+    "node_modules/tldts-core": {
+      "version": "5.7.52",
+      "resolved": "https://registry.npmjs.org/tldts-core/-/tldts-core-5.7.52.tgz",
+      "integrity": "sha512-xLa9sSdmSIec2kok76Fv68vvf58QZJFGEOan/pLsk9rrPfkfMZq7Vr03X0hPr8VCc09tcbfU6izXzRDl8oItYQ=="
+    },
     "node_modules/tmp": {
       "version": "0.0.33",
       "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.0.33.tgz",
@@ -9825,6 +9842,7 @@
         "rimraf": "^3.0.2",
         "rxjs": "6.6.7",
         "tldjs": "^2.3.1",
+        "tldts": "5.7.52",
         "typescript": "4.1.5",
         "zxcvbn": "^4.4.2"
       }
@@ -15747,6 +15765,19 @@
         }
       }
     },
+    "tldts": {
+      "version": "5.7.52",
+      "resolved": "https://registry.npmjs.org/tldts/-/tldts-5.7.52.tgz",
+      "integrity": "sha512-0ddIYX/04P8qFVY0XYAlF7vJEef+nd+natGSnlasB8MRC4iZIz5HzX4G0Vm6jRItxEtWUuXo2+eeBLBtsZZgEQ==",
+      "requires": {
+        "tldts-core": "^5.7.52"
+      }
+    },
+    "tldts-core": {
+      "version": "5.7.52",
+      "resolved": "https://registry.npmjs.org/tldts-core/-/tldts-core-5.7.52.tgz",
+      "integrity": "sha512-xLa9sSdmSIec2kok76Fv68vvf58QZJFGEOan/pLsk9rrPfkfMZq7Vr03X0hPr8VCc09tcbfU6izXzRDl8oItYQ=="
+    },
     "tmp": {
       "version": "0.0.33",
       "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.0.33.tgz",