tldts - npm Package Compare versions

Comparing version 4.0.6 to 5.0.0

CHANGELOG.md

@@ -5,2 +5,37 @@ ## Change Log
 
+### 5.0.0
+
+*2019-05-23*
+
+- Improvements in various areas [#149](https://github.com/remusao/tldts/pull/149)
+  * Performance improvements in all methods of public API (up to x2 faster)
+    * `extractHostname`: will now avoid lower-casing the result in some cases
+    * `extractHostname`: handles single or triple '/' after protocol
+    * `extractHostname`: has fast-path for validation of common protocols (e.g. https)
+    * `isProbablyIpv4`: performs first quick check on length of hostname
+    * `isProbablyIpv6`: performs first quick check on length of hostname
+    * `isValidHostname`: make use of `charCodeAt` instead of `codePointAt`
+    * `lookupInTrie`: makes use of Trie with more stable structure (faster)
+    * `lookupInTrie`: lazily allocate memory for result
+    * `suffixLookup`: uses fast-path for most common suffixes (massive speed-up)
+    * `suffixLookup`: does not allocate memory for result anymore
+    * `setDefaults`: fast-path in case no argument was provided
+    * `getSubdomain`: fast-path if subdomain is empty
+  * Add more options to fine-tune behavior and performance
+    * `detectIp` allows to disable IP check
+    * `mixedInput` allows to specify if we expect a mix of URLs and hostnames as
+      input. If only hostnames are expected then `extractHostname` can be set to
+      `false` to speed-up parsing. If only URLs are expected then `mixedInputs`
+      can be set to `false`. The `mixedInputs` is only a hint and will not
+      change the behavior of the library.
+    * `validateHostname` can be set to `false` to disable validation and
+      speed-up processing further.
+  * Check that input is string before parsing
+  * Fix support for reserved keywords in hostnames
+  * Add tests and bring back coverage to 100%
+  * Minified bundle is now also tested with the same suite
+  * Migrate utils scripts from `bin/` folder to TypeScript
+  * Add small `tldts` cli which can be used to parse URLs
+  * Update README with more accurate information
+
 ### 4.0.6
@@ -7,0 +42,0 @@

dist/lib/factory.d.ts

@@ -5,3 +5,3 @@ import { IPublicSuffix, ISuffixLookupOptions } from './lookup/interface';
     hostname: string | null;
-    isIp: boolean;
+    isIp: boolean | null;
     subdomain: string | null;
@@ -21,2 +21,2 @@ domain: string | null;
 }
-export default function parseImpl(url: string, step: FLAG, suffixLookup: (_1: string, _2: ISuffixLookupOptions) => IPublicSuffix | null, partialOptions?: Partial<IOptions>): IResult;
+export default function parseImpl(url: string, step: FLAG, suffixLookup: (_1: string, _2: ISuffixLookupOptions, _3: IPublicSuffix) => void, partialOptions?: Partial<IOptions>): IResult;

dist/lib/lookup/interface.d.ts

 export interface IPublicSuffix {
-    isIcann: boolean;
-    isPrivate: boolean;
+    isIcann: boolean | null;
+    isPrivate: boolean | null;
     publicSuffix: string | null;
@@ -5,0 +5,0 @@ }

dist/lib/lookup/packed-hashes.d.ts

 import { IPublicSuffix, ISuffixLookupOptions } from './interface';
-export default function suffixLookup(hostname: string, { allowIcannDomains, allowPrivateDomains }: ISuffixLookupOptions): IPublicSuffix | null;
+export default function suffixLookup(hostname: string, { allowIcannDomains, allowPrivateDomains }: ISuffixLookupOptions, out: IPublicSuffix): void;

dist/lib/lookup/suffix-trie.d.ts

 import { IPublicSuffix, ISuffixLookupOptions } from './interface';
-export default function suffixLookup(hostname: string, options: ISuffixLookupOptions): IPublicSuffix | null;
+export default function suffixLookup(hostname: string, options: ISuffixLookupOptions, out: IPublicSuffix): void;

dist/lib/options.d.ts

 export interface IOptions {
     allowIcannDomains: boolean;
     allowPrivateDomains: boolean;
+    detectIp: boolean;
     extractHostname: boolean;
+    mixedInputs: boolean;
     validHosts: string[] | null;
+    validateHostname: boolean;
 }
-export declare function setDefaults({ allowIcannDomains, allowPrivateDomains, extractHostname, validHosts, }?: Partial<IOptions>): IOptions;
+export declare function setDefaults(options?: Partial<IOptions>): IOptions;

lib/extract-hostname.ts

@@ -9,4 +9,5 @@ /**
 ): string | null {
-  let start = 0;
-  let end = url.length;
+  let start: number = 0;
+  let end: number = url.length;
+  let hasUpper: boolean = false;
 
@@ -26,34 +27,82 @@ // If url is not already a valid hostname, then try to extract hostname.
     // Skip scheme.
-    if (url.charCodeAt(start) === 47 && url.charCodeAt(start + 1) === 47) {
-      // check if url starts with '//'
+    if (
+      url.charCodeAt(start) === 47 /* '/' */ &&
+      url.charCodeAt(start + 1) === 47 /* '/' */
+    ) {
       start += 2;
     } else {
-      const indexOfProtocol = url.indexOf('://', start);
+      const indexOfProtocol = url.indexOf(':/', start);
       if (indexOfProtocol !== -1) {
-        start = indexOfProtocol + 3;
+        // Implement fast-path for common protocols. We expect most protocols
+        // should be one of these 4 and thus we will not need to perform the
+        // more expansive validity check most of the time.
+        const protocolSize = indexOfProtocol - start;
+        const c0 = url.charCodeAt(start);
+        const c1 = url.charCodeAt(start + 1);
+        const c2 = url.charCodeAt(start + 2);
+        const c3 = url.charCodeAt(start + 3);
+        const c4 = url.charCodeAt(start + 4);
 
-        // Check that scheme is valid
-        for (let i = 0; i < indexOfProtocol; i += 1) {
-          const lowerCaseCode = url.charCodeAt(i) | 32;
-          if (
-            ((lowerCaseCode >= 97 && lowerCaseCode <= 122) || // [a, z]
-            (lowerCaseCode >= 48 && lowerCaseCode <= 57) || // [0, 9]
-            lowerCaseCode === 46 || // '.'
-            lowerCaseCode === 45 || // '-'
-              lowerCaseCode === 43) === false // '+'
-          ) {
-            return null;
+        if (
+          protocolSize === 5 &&
+          c0 === 104 /* 'h' */ &&
+          c1 === 116 /* 't' */ &&
+          c2 === 116 /* 't' */ &&
+          c3 === 112 /* 'p' */ &&
+          c4 === 115 /* 's' */
+        ) {
+          // https
+        } else if (
+          protocolSize === 4 &&
+          c0 === 104 /* 'h' */ &&
+          c1 === 116 /* 't' */ &&
+          c2 === 116 /* 't' */ &&
+          c3 === 112 /* 'p' */
+        ) {
+          // http
+        } else if (
+          protocolSize === 3 &&
+          c0 === 119 /* 'w' */ &&
+          c1 === 115 /* 's' */ &&
+          c2 === 115 /* 's' */
+        ) {
+          // wss
+        } else if (
+          protocolSize === 2 &&
+          c0 === 119 /* 'w' */ &&
+          c1 === 115 /* 's' */
+        ) {
+          // ws
+        } else {
+          // Check that scheme is valid
+          for (let i = start; i < indexOfProtocol; i += 1) {
+            const lowerCaseCode = url.charCodeAt(i) | 32;
+            if (
+              ((lowerCaseCode >= 97 && lowerCaseCode <= 122) || // [a, z]
+              (lowerCaseCode >= 48 && lowerCaseCode <= 57) || // [0, 9]
+              lowerCaseCode === 46 || // '.'
+              lowerCaseCode === 45 || // '-'
+                lowerCaseCode === 43) === false // '+'
+            ) {
+              return null;
+            }
           }
         }
+
+        // Skip 0, 1 or more '/' after ':/'
+        start = indexOfProtocol + 2;
+        while (url.charCodeAt(start) === 47 /* '/' */) {
+          start += 1;
+        }
       }
     }
 
-    // Detect first occurrence of '/', '?' or '#'. We also keep track of the last
-    // occurrence of '@', ']' or ':' to speed-up subsequent parsing of
+    // Detect first occurrence of '/', '?' or '#'. We also keep track of the
+    // last occurrence of '@', ']' or ':' to speed-up subsequent parsing of
     // (respectively), identifier, ipv6 or port.
-    let indexOfIdentifier = -1;
-    let indexOfClosingBracket = -1;
-    let indexOfPort = -1;
+    let indexOfIdentifier: number = -1;
+    let indexOfClosingBracket: number = -1;
+    let indexOfPort: number = -1;
     for (let i = start; i < end; i += 1) {
-      const code = url.charCodeAt(i);
+      const code: number = url.charCodeAt(i);
       if (
@@ -75,2 +124,4 @@ code === 35 || // '#'
         indexOfPort = i;
+      } else if (code >= 65 && code <= 90) {
+        hasUpper = true;
       }
@@ -89,4 +140,3 @@ }
     // Handle ipv6 addresses
-    if (url.charCodeAt(start) === 91) {
-      // '['
+    if (url.charCodeAt(start) === 91 /* '[' */) {
       if (indexOfClosingBracket !== -1) {
@@ -103,11 +153,14 @@ return url.slice(start + 1, indexOfClosingBracket).toLowerCase();
   // Trim trailing dots
-  while (end > start + 1 && url.charCodeAt(end - 1) === 46) {
-    // '.'
+  while (end > start + 1 && url.charCodeAt(end - 1) === 46 /* '.' */) {
     end -= 1;
   }
 
-  return (start !== 0 || end !== url.length
-    ? url.slice(start, end)
-    : url
-  ).toLowerCase();
+  const hostname: string =
+    start !== 0 || end !== url.length ? url.slice(start, end) : url;
+
+  if (hasUpper) {
+    return hostname.toLowerCase();
+  }
+
+  return hostname;
 }

lib/factory.ts

@@ -13,3 +13,2 @@ /**
 import { IOptions, setDefaults } from './options';
-import getPublicSuffix from './public-suffix';
 import getSubdomain from './subdomain';
@@ -25,3 +24,3 @@
   // Is `hostname` an IP? (IPv4 or IPv6)
-  isIp: boolean;
+  isIp: boolean | null;
 
@@ -54,3 +53,7 @@ // `hostname` split between subdomain, domain and its public suffix (if any)
   step: FLAG,
-  suffixLookup: (_1: string, _2: ISuffixLookupOptions) => IPublicSuffix | null,
+  suffixLookup: (
+    _1: string,
+    _2: ISuffixLookupOptions,
+    _3: IPublicSuffix,
+  ) => void,
   partialOptions?: Partial<IOptions>,
@@ -63,3 +66,3 @@ ): IResult {
     isIcann: null,
-    isIp: false,
+    isIp: null,
     isPrivate: null,
@@ -70,42 +73,52 @@ publicSuffix: null,
 
-  // Extract hostname from `url` only if needed
+  // Very fast approximate check to make sure `url` is a string. This is needed
+  // because the library will not necessarily be used in a typed setup and
+  // values of arbitrary types might be given as argument.
+  if (typeof url !== 'string') {
+    return result;
+  }
+
+  // Extract hostname from `url` only if needed. This can be made optional
+  // using `options.extractHostname`. This option will typically be used
+  // whenever we are sure the inputs to `parse` are already hostnames and not
+  // arbitrary URLs.
+  //
+  // `mixedInput` allows to specify if we expect a mix of URLs and hostnames
+  // as input. If only hostnames are expected then `extractHostname` can be
+  // set to `false` to speed-up parsing. If only URLs are expected then
+  // `mixedInputs` can be set to `false`. The `mixedInputs` is only a hint
+  // and will not change the behavior of the library.
   if (options.extractHostname === false) {
     result.hostname = url;
-
-    if (step === FLAG.HOSTNAME) {
-      return result;
-    }
+  } else if (options.mixedInputs === true) {
+    result.hostname = extractHostname(url, isValidHostname(url));
   } else {
-    const urlIsValidHostname = isValidHostname(url);
-    result.hostname = extractHostname(url, urlIsValidHostname);
+    result.hostname = extractHostname(url, false);
+  }
 
-    if (step === FLAG.HOSTNAME || result.hostname === null) {
-      return result;
-    }
+  if (step === FLAG.HOSTNAME || result.hostname === null) {
+    return result;
+  }
 
-    // Check if `hostname` is a valid ip address
+  // Check if `hostname` is a valid ip address
+  if (options.detectIp === true) {
     result.isIp = isIp(result.hostname);
-    if (result.isIp) {
+    if (result.isIp === true) {
       return result;
     }
+  }
 
-    // Make sure hostname is valid before proceeding
-    if (
-      urlIsValidHostname === false &&
-      isValidHostname(result.hostname) === false
-    ) {
-      return result;
-    }
+  // Perform optional hostname validation. If hostname is not valid, no need to
+  // go further as there will be no valid domain or sub-domain.
+  if (
+    options.validateHostname === true &&
+    options.extractHostname === true &&
+    isValidHostname(result.hostname) === false
+  ) {
+    result.hostname = null;
+    return result;
   }
 
   // Extract public suffix
-  const publicSuffixResult = getPublicSuffix(
-    result.hostname,
-    options,
-    suffixLookup,
-  );
-
-  result.publicSuffix = publicSuffixResult.publicSuffix;
-  result.isIcann = publicSuffixResult.isIcann;
-  result.isPrivate = publicSuffixResult.isIcann === false;
+  suffixLookup(result.hostname, options, result);
   if (step === FLAG.PUBLIC_SUFFIX || result.publicSuffix === null) {
@@ -112,0 +125,0 @@ return result;

lib/is-ip.ts

@@ -6,2 +6,12 @@ /**
 function isProbablyIpv4(hostname: string): boolean {
+  // Cannot be shorted than 1.1.1.1
+  if (hostname.length < 7) {
+    return false;
+  }
+
+  // Cannot be longer than: 255.255.255.255
+  if (hostname.length > 15) {
+    return false;
+  }
+
   let numberOfDots = 0;
@@ -12,8 +22,5 @@
 
-    if (code === 46) {
-      // '.'
+    if (code === 46 /* '.' */) {
       numberOfDots += 1;
-    } else if (code < 48 || code > 57) {
-      // 48 => '0'
-      // 57 => '9'
+    } else if (code < 48 /* '0' */ || code > 57 /* '9' */) {
       return false;
@@ -25,4 +32,4 @@ }
     numberOfDots === 3 &&
-    hostname[0] !== '.' &&
-    hostname[hostname.length - 1] !== '.'
+    hostname.charCodeAt(0) !== 46 /* '.' */ &&
+    hostname.charCodeAt(hostname.length - 1) !== 46 /* '.' */
   );
@@ -35,9 +42,15 @@ }
 function isProbablyIpv6(hostname: string): boolean {
-  let hasColon = false;
+  // We only consider the maximum size of a normal IPV6. Note that this will
+  // fail on so-called "IPv4 mapped IPv6 addresses" but this is a corner-case
+  // and a proper validation library should be used for these.
+  if (hostname.length > 39) {
+    return false;
+  }
 
+  let hasColon: boolean = false;
+
   for (let i = 0; i < hostname.length; i += 1) {
     const code = hostname.charCodeAt(i);
 
-    if (code === 58) {
-      // ':'
+    if (code === 58 /* ':' */) {
       hasColon = true;
@@ -44,0 +57,0 @@ } else if (

lib/is-valid.ts

@@ -31,4 +31,3 @@ /**
 
-  // @ts-ignore
-  if (!isValidAscii(hostname.codePointAt(0))) {
+  if (!isValidAscii(hostname.charCodeAt(0))) {
     return false;
@@ -38,10 +37,9 @@ }
   // Validate hostname according to RFC
-  let lastDotIndex = -1;
-  let lastCharCode;
+  let lastDotIndex: number = -1;
+  let lastCharCode: number = -1;
   const len = hostname.length;
 
   for (let i = 0; i < len; i += 1) {
-    const code = hostname.codePointAt(i);
-    if (code === 46) {
-      // '.'
+    const code = hostname.charCodeAt(i);
+    if (code === 46 /* '.' */) {
       if (
@@ -61,3 +59,2 @@ // Check that previous label is < 63 bytes long (64 = 63 + '.')
       lastDotIndex = i;
-      // @ts-ignore
     } else if (!(isValidAscii(code) || code === 45 || code === 95)) {
@@ -64,0 +61,0 @@ // Check if there is a forbidden character in the label

lib/lookup/interface.ts

 export interface IPublicSuffix {
-  isIcann: boolean;
-  isPrivate: boolean;
+  isIcann: boolean | null;
+  isPrivate: boolean | null;
   publicSuffix: string | null;
@@ -5,0 +5,0 @@ }

lib/lookup/packed-hashes.ts

@@ -5,2 +5,15 @@ import packed from './data/hashes';
 /**
+ * Utility to extract the TLD from a hostname string
+ */
+function extractTldFromHost(hostname: string): string | null {
+  const lastDotIndex = hostname.lastIndexOf('.');
+  if (lastDotIndex === -1) {
+    // Single label is considered a tld
+    return hostname;
+  }
+
+  return hostname.slice(lastDotIndex + 1);
+}
+
+/**
  * Find `elt` in `arr` between indices `start` (included) and `end` (excluded)
@@ -92,3 +105,4 @@ * using a binary search algorithm.
   { allowIcannDomains, allowPrivateDomains }: ISuffixLookupOptions,
-): IPublicSuffix | null {
+  out: IPublicSuffix,
+): void {
   // Keep track of longest match
@@ -224,3 +238,6 @@ let matchIndex = -1;
     // No match found
-    return null;
+    out.isIcann = false;
+    out.isPrivate = false;
+    out.publicSuffix = extractTldFromHost(hostname);
+    return;
   } else if ((matchKind & Result.EXCEPTION_MATCH) !== 0) {
@@ -247,7 +264,5 @@ // If match is an exception, this means that we need to count less label.
 
-  return {
-    isIcann: (matchKind & Result.ICANN_MATCH) !== 0,
-    isPrivate: (matchKind & Result.PRIVATE_MATCH) !== 0,
-    publicSuffix,
-  };
+  out.isIcann = (matchKind & Result.ICANN_MATCH) !== 0;
+  out.isPrivate = (matchKind & Result.PRIVATE_MATCH) !== 0;
+  out.publicSuffix = publicSuffix;
 }

lib/lookup/suffix-trie.ts

@@ -1,2 +0,2 @@
-import { exceptions, rules } from './data/trie';
+import { exceptions, ITrie, rules } from './data/trie';
 import { IPublicSuffix, ISuffixLookupOptions } from './interface';
@@ -13,2 +13,3 @@
   isIcann: boolean;
+  isPrivate: boolean;
 }
@@ -21,17 +22,16 @@
   parts: string[],
-  trie: any,
+  trie: ITrie,
   index: number,
   allowedMask: number,
-): IMatch {
-  let node = trie;
-  const lookupResult: IMatch = {
-    index: -1,
-    isIcann: false,
-  };
-
+): IMatch | null {
+  let result: IMatch | null = null;
+  let node: ITrie | undefined = trie;
   while (node !== undefined) {
     // We have a match!
-    if (node.$ !== undefined && (node.$ & allowedMask) !== 0) {
-      lookupResult.index = index + 1;
-      lookupResult.isIcann = node.$ === RULE_TYPE.ICANN;
+    if ((node.$ & allowedMask) !== 0) {
+      result = {
+        index: index + 1,
+        isIcann: node.$ === RULE_TYPE.ICANN,
+        isPrivate: node.$ === RULE_TYPE.PRIVATE,
+      };
     }
@@ -41,10 +41,13 @@
     if (index === -1) {
-      return lookupResult;
+      break;
     }
 
-    node = node[parts[index]] || node['*'];
+    const succ: {
+      [label: string]: ITrie;
+    } = node.succ;
+    node = succ && (succ[parts[index]] || succ['*']);
     index -= 1;
   }
 
-  return lookupResult;
+  return result;
 }
@@ -58,21 +61,85 @@
   options: ISuffixLookupOptions,
-): IPublicSuffix | null {
-  const allowIcannDomains = options.allowIcannDomains;
-  const allowPrivateDomains = options.allowPrivateDomains;
+  out: IPublicSuffix,
+): void {
+  // Fast path for very popular suffixes; this allows to by-pass lookup
+  // completely as well as any extra allocation or string manipulation.
+  if (options.allowPrivateDomains === false && hostname.length > 3) {
+    const last: number = hostname.length - 1;
+    const c3: number = hostname.charCodeAt(last);
+    const c2: number = hostname.charCodeAt(last - 1);
+    const c1: number = hostname.charCodeAt(last - 2);
+    const c0: number = hostname.charCodeAt(last - 3);
+
+    if (
+      c3 === 109 /* 'm' */ &&
+      c2 === 111 /* 'o' */ &&
+      c1 === 99 /* 'c' */ &&
+      c0 === 46 /* '.' */
+    ) {
+      out.isIcann = true;
+      out.isPrivate = false;
+      out.publicSuffix = 'com';
+      return;
+    } else if (
+      c3 === 103 /* 'g' */ &&
+      c2 === 114 /* 'r' */ &&
+      c1 === 111 /* 'o' */ &&
+      c0 === 46 /* '.' */
+    ) {
+      out.isIcann = true;
+      out.isPrivate = false;
+      out.publicSuffix = 'org';
+      return;
+    } else if (
+      c3 === 117 /* 'u' */ &&
+      c2 === 100 /* 'd' */ &&
+      c1 === 101 /* 'e' */ &&
+      c0 === 46 /* '.' */
+    ) {
+      out.isIcann = true;
+      out.isPrivate = false;
+      out.publicSuffix = 'edu';
+      return;
+    } else if (
+      c3 === 118 /* 'v' */ &&
+      c2 === 111 /* 'o' */ &&
+      c1 === 103 /* 'g' */ &&
+      c0 === 46 /* '.' */
+    ) {
+      out.isIcann = true;
+      out.isPrivate = false;
+      out.publicSuffix = 'gov';
+      return;
+    } else if (
+      c3 === 116 /* 't' */ &&
+      c2 === 101 /* 'e' */ &&
+      c1 === 110 /* 'n' */ &&
+      c0 === 46 /* '.' */
+    ) {
+      out.isIcann = true;
+      out.isPrivate = false;
+      out.publicSuffix = 'net';
+      return;
+    } else if (
+      c3 === 101 /* 'e' */ &&
+      c2 === 100 /* 'd' */ &&
+      c1 === 46 /* '.' */
+    ) {
+      out.isIcann = true;
+      out.isPrivate = false;
+      out.publicSuffix = 'de';
+      return;
+    }
+  }
+
   const hostnameParts = hostname.split('.');
 
-  let allowedMask = 0;
+  const allowedMask =
+    (options.allowPrivateDomains === true ? RULE_TYPE.PRIVATE : 0) |
+    (options.allowIcannDomains === true ? RULE_TYPE.ICANN : 0);
 
-  // Define set of accepted public suffix (ICANN, PRIVATE)
-  if (allowPrivateDomains === true) {
-    allowedMask |= RULE_TYPE.PRIVATE;
-  }
-  if (allowIcannDomains === true) {
-    allowedMask |= RULE_TYPE.ICANN;
-  }
-
-  // Look for a match in rules
-  const lookupResult = lookupInTrie(
+  // Look for exceptions
+  const exceptionMatch = lookupInTrie(
     hostnameParts,
-    rules,
+    exceptions,
     hostnameParts.length - 1,
@@ -82,10 +149,13 @@ allowedMask,
 
-  if (lookupResult.index === -1) {
-    return null;
+  if (exceptionMatch !== null) {
+    out.isIcann = exceptionMatch.isIcann;
+    out.isPrivate = exceptionMatch.isPrivate;
+    out.publicSuffix = hostnameParts.slice(exceptionMatch.index + 1).join('.');
+    return;
   }
 
-  // Look for exceptions
-  const exceptionLookupResult = lookupInTrie(
+  // Look for a match in rules
+  const rulesMatch = lookupInTrie(
     hostnameParts,
-    exceptions,
+    rules,
     hostnameParts.length - 1,
@@ -95,17 +165,15 @@ allowedMask,
 
-  if (exceptionLookupResult.index !== -1) {
-    return {
-      isIcann: exceptionLookupResult.isIcann,
-      isPrivate: !exceptionLookupResult.isIcann,
-      publicSuffix: hostnameParts
-        .slice(exceptionLookupResult.index + 1)
-        .join('.'),
-    };
+  if (rulesMatch !== null) {
+    out.isIcann = rulesMatch.isIcann;
+    out.isPrivate = rulesMatch.isPrivate;
+    out.publicSuffix = hostnameParts.slice(rulesMatch.index).join('.');
+    return;
   }
 
-  return {
-    isIcann: lookupResult.isIcann,
-    isPrivate: !lookupResult.isIcann,
-    publicSuffix: hostnameParts.slice(lookupResult.index).join('.'),
-  };
+  // No match found...
+  // Prevailing rule is '*' so we consider the top-level domain to be the
+  // public suffix of `hostname` (e.g.: 'example.org' => 'org').
+  out.isIcann = false;
+  out.isPrivate = false;
+  out.publicSuffix = hostnameParts[hostnameParts.length - 1];
 }

lib/options.ts

 export interface IOptions {
   allowIcannDomains: boolean;
   allowPrivateDomains: boolean;
+  detectIp: boolean;
   extractHostname: boolean;
+  mixedInputs: boolean;
   validHosts: string[] | null;
+  validateHostname: boolean;
 }
 
-export function setDefaults({
+function setDefaultsImpl({
   allowIcannDomains = true,
   allowPrivateDomains = false,
+  detectIp = true,
   extractHostname = true,
+  mixedInputs = true,
   validHosts = null,
-}: Partial<IOptions> = {}): IOptions {
+  validateHostname = true,
+}: Partial<IOptions>): IOptions {
   return {
     allowIcannDomains,
     allowPrivateDomains,
+    detectIp,
     extractHostname,
+    mixedInputs,
     validHosts,
+    validateHostname,
   };
 }
+
+const DEFAULT_OPTIONS = setDefaultsImpl({});
+
+export function setDefaults(options?: Partial<IOptions>): IOptions {
+  if (options === undefined) {
+    return DEFAULT_OPTIONS;
+  }
+
+  return setDefaultsImpl(options);
+}

lib/subdomain.ts

@@ -5,3 +5,8 @@ /**
 export default function getSubdomain(hostname: string, domain: string): string {
+  // If `hostname` and `domain` are the same, then there is no sub-domain
+  if (domain.length === hostname.length) {
+    return '';
+  }
+
   return hostname.slice(0, -domain.length - 1);
 }

package.json

 {
   "name": "tldts",
   "description": "Library to work against complex domain names, subdomains and URIs.",
-  "version": "4.0.6",
+  "version": "5.0.0",
   "homepage": "https://github.com/remusao/tldts",
@@ -20,5 +20,5 @@ "author": "Rémi Berson",
   },
-  "browser": "dist/tldts.umd.js",
-  "main": "dist/tldts.cjs.js",
-  "module": "dist/tldts.esm.js",
+  "browser": "dist/tldts.umd.min.js",
+  "main": "dist/tldts.cjs.min.js",
+  "module": "dist/tldts.esm.min.js",
   "types": "dist/tldts.d.ts",
@@ -32,16 +32,16 @@ "files": [
   "license": "MIT",
+  "bin": {
+    "tldts": "bin/cli.js"
+  },
   "scripts": {
     "clean": "rm -rfv dist build",
     "lint": "tslint -c tslint.json 'lib/**/*.ts'",
-    "update": "git submodule foreach git pull origin master && node ./bin/update.js",
+    "update": "git submodule foreach git pull origin master && ts-node -O '{\"module\": \"commonjs\"}' ./bin/update.ts",
     "build": "tsc -p .",
     "prebuild": "npm run clean",
-    "bundle": "rollup -c rollup.config.js",
+    "bundle": "rollup -c rollup.config.ts",
     "prebundle": "npm run build",
-    "minify-tldts": "google-closure-compiler --js=./dist/tldts.umd.js --js_output_file=./dist/tldts.umd.min.js",
-    "minify-tldts-experimental": "google-closure-compiler --js=./dist/tldts-experimental.umd.js --js_output_file=./dist/tldts-experimental.umd.min.js",
-    "minify": "npm run minify-tldts && npm run minify-tldts-experimental",
-    "preminify": "npm run bundle",
-    "prepack": "npm run minify",
+    "prepack": "npm run bundle",
     "test": "jest --coverage --no-cache ./test/",
+    "pretest": "npm pack",
     "benchmark": "make -C bench",
@@ -52,17 +52,14 @@ "prebenchmark": "npm pack"
   "devDependencies": {
-    "@types/jest": "^24.0.11",
-    "@types/node": "^11.12.1",
+    "@ampproject/rollup-plugin-closure-compiler": "^0.9.0",
+    "@types/jest": "^24.0.12",
+    "@types/node": "^12.0.2",
     "benchmark": "^2.1.4",
+    "chalk": "^2.4.2",
     "coveralls": "^3.0.3",
-    "eslint": "^5.15.3",
-    "eslint-config-airbnb": "^17.1.0",
-    "eslint-plugin-import": "^2.16.0",
-    "eslint-plugin-jsx-a11y": "^6.2.1",
-    "eslint-plugin-react": "^7.12.4",
-    "google-closure-compiler": "^20190325.0.0",
-    "jest": "^24.5.0",
-    "rollup": "^1.7.4",
-    "ts-jest": "^24.0.0",
-    "tslint": "^5.14.0",
-    "typescript": "^3.3.4000"
+    "jest": "^24.8.0",
+    "rollup": "^1.11.3",
+    "ts-jest": "^24.0.2",
+    "ts-node": "^8.1.0",
+    "tslint": "^5.16.0",
+    "typescript": "^3.4.5"
   },
@@ -81,4 +78,5 @@ "keywords": [
     "public suffix",
+    "url parsing",
     "typescript"
   ]
 }

README.md

@@ -1,2 +0,2 @@
-# tldts - Hostname and Domain Parsing using Public Suffix Lists
+# tldts - Blazing Fast URL Parsing
 
@@ -9,15 +9,14 @@ [![NPM](https://nodei.co/npm/tldts.png?downloads=true&downloadRank=true)](https://nodei.co/npm/tldts/)
 
+`tldts` is a JavaScript library to extract hostnames, domains, public suffixes, top-level domains and subdomains from URLs.
 
-> `tldts` is a Typescript library to parse hostnames, domains, public suffixes, top-level domains and subdomains from URLs.
-
-
 **Features**:
-1. **Fastest library** around (up to 2M operations per second, that's 3 orders of
-   magnitude faster than the most popular library out there)
-2. Written in **TypeScript**, ships with `umd`, `esm`, `cjs` bundles and *type definitions*
+1. Tuned for **performance** (order of 0.1 to 1 μs per input)
+2. Handles both URLs and hostnames
 3. Full Unicode/IDNA support
-4. Support both ICANN and Private suffixes
-5. Ships with continuously updated version of the list: it works *out of the box*!
-6. Support parsing full URLs or hostnames
-7. Small bundles and small memory footprint
+4. Support parsing email addresses
+5. Detect IPv4 and IPv6 addresses
+6. Continuously updated version of the public suffix list
+7. **TypeScript**, ships with `umd`, `esm`, `cjs` bundles and *type definitions*
+8. Small bundles and small memory footprint
+9. Battle tested: full test coverage and production use
 
@@ -33,3 +32,3 @@ # Install
 ```js
-const tldts = require('tldts');
+const { parse } = require('tldts');
 
@@ -47,2 +46,8 @@ // Retrieving hostname related informations of a given URL
 
+Modern *ES6 modules import* is also supported:
+
+```js
+import { parse } from 'tldts';
+```
+
 # API
@@ -57,3 +62,5 @@
 The behavior of `tldts` can be customized using an `options` argument for all
-the functions exposed as part of the public API.
+the functions exposed as part of the public API. This is useful to both change
+the behavior of the library as well as fine-tune the performance depending on
+your inputs.
 
@@ -67,3 +74,15 @@ ```js
   // Extract and validate hostname (default: true)
+  // When set to `false`, inputs will be considered valid hostnames.
   extractHostname: boolean;
+  // Validate hostnames after parsing (default: true)
+  // If a hostname is not valid, not further processing is performed. When set
+  // to `false`, inputs to the library will be considered valid and parsing will
+  // proceed regardless.
+  validateHostname: boolean;
+  // Perform IP address detection (default: true).
+  detectIp: boolean;
+  // Assume that both URLs and hostnames can be given as input (default: true)
+  // If set to `false` we assume only URLs will be given as input, which
+  // speed-ups processing.
+  mixedInputs: boolean;
   // Specifies extra valid suffixes (default: null)
@@ -70,0 +89,0 @@ validHosts: string[] | null;

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Major refactor

Supply chain risk

Package has recently undergone a major refactor. It may be unstable or indicate significant internal changes. Use caution when updating to versions that include significant changes.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Dev dependency count

12

decreased by-20%

Number of package files

45

increased by7.14%

Number of lines in readme file

286

increased by7.12%

Worsened metrics

Total package byte prevSize

1574685

decreased by-6.74%

Lines of code

8693

decreased by-74.25%

Number of medium supply chain risk alerts

1

increased byInfinity%

No dependency changes