cacheable-lookup - npm Package Compare versions

Comparing version 4.1.2 to 4.2.0

index.d.ts

@@ -1,2 +0,2 @@
-import {Resolver, LookupAddress, promises as dnsPromises} from 'dns';
+import {Resolver, promises as dnsPromises} from 'dns';
 import {Agent} from 'http';
@@ -6,3 +6,3 @@
 
-type IPFamily = 4 | 6;
+export type IPFamily = 4 | 6;
 
@@ -39,5 +39,19 @@ type TPromise<T> = T | Promise<T>;
 	customHostsPath?: string | false;
+	/**
+	 * The lifetime of the entries received from the OS (TTL in seconds).
+	 *
+	 * **Note**: This option is independent, `options.maxTtl` does not affect this.
+	 * @default 1
+	 */
+	fallbackTtl?: number;
+	/**
+	 * The time how long it needs to remember failed queries (TTL in seconds).
+	 *
+	 * **Note**: This option is independent, `options.maxTtl` does not affect this.
+	 * @default 0.15
+	 */
+	errorTtl?: number;
 }
 
-interface EntryObject {
+export interface EntryObject {
 	/**
@@ -61,3 +75,3 @@ * The IP address (can be an IPv4 or IPv5 address).
 
-interface LookupOptions {
+export interface LookupOptions {
 	/**
@@ -78,10 +92,2 @@ * One or more supported getaddrinfo flags. Multiple flags may be passed by bitwise ORing their values.
 
-interface AsyncLookupOptions extends LookupOptions {
-	/**
-	 * Throw when there's no match. If set to `false` and it gets no match, it will return `undefined`.
-	 * @default false
-	 */
-	throwNotFound?: boolean;
-}
-
 export default class CacheableLookup {
@@ -103,4 +109,4 @@ constructor(options?: Options);
 	 */
-	lookupAsync(hostname: string, options: AsyncLookupOptions & {all: true}): Promise<ReadonlyArray<EntryObject>>;
-	lookupAsync(hostname: string, options: AsyncLookupOptions): Promise<EntryObject>;
+	lookupAsync(hostname: string, options: LookupOptions & {all: true}): Promise<ReadonlyArray<EntryObject>>;
+	lookupAsync(hostname: string, options: LookupOptions): Promise<EntryObject>;
 	lookupAsync(hostname: string): Promise<EntryObject>;
@@ -117,2 +123,7 @@ lookupAsync(hostname: string, family: IPFamily): Promise<EntryObject>;
 	/**
+	 * Returns an entry from the array for the given hostname.
+	 * Useful to implement a round-robin algorithm.
+	 */
+	_getEntry(entries: ReadonlyArray<EntryObject>, hostname: string): EntryObject;
+	/**
 	 * Removes outdated entries.
@@ -134,5 +145,5 @@ */
 	/**
-	 * Clears the cache.
+	 * Clears the cache for the given hostname. If the hostname argument is not present, the entire cache will be cleared.
 	 */
-	clear(): void;
+	clear(hostname?: string): void;
 }

package.json

 {
   "name": "cacheable-lookup",
-  "version": "4.1.2",
+  "version": "4.2.0",
   "description": "A cacheable dns.lookup(…) that respects the TTL",
@@ -35,3 +35,3 @@ "engines": {
     "@types/keyv": "^3.1.1",
-    "ava": "^3.1.0",
+    "ava": "^3.7.1",
     "benchmark": "^2.1.4",
@@ -38,0 +38,0 @@ "coveralls": "^3.0.9",

README.md

@@ -48,4 +48,4 @@ # cacheable-lookup
 
-Type: [`TTLMap`](index.d.ts) | [`Keyv`](https://github.com/lukechilds/keyv/)<br>
-Default: `new TTLMap()`
+Type: `Map` | [`Keyv`](https://github.com/lukechilds/keyv/)<br>
+Default: `new Map()`
 
@@ -68,6 +68,30 @@ Custom cache instance. If `undefined`, it will create a new one.
 
-Limits the cache time (TTL in seconds).
+The maximum lifetime of the entries received from the specifed DNS server (TTL in seconds).
 
 If set to `0`, it will make a new DNS query each time.
 
+**Pro Tip**: This shouldn't be lower than your DNS server response time in order to prevent bottlenecks. For example, if you use Cloudflare, this value should be greater than `0.01`.
+
+##### options.fallbackTtl
+
+Type: `number`<br>
+Default: `1`
+
+The lifetime of the entries received from the OS (TTL in seconds).
+
+**Note**: This option is independent, `options.maxTtl` does not affect this.
+
+**Pro Tip**: This shouldn't be lower than your DNS server response time in order to prevent bottlenecks. For example, if you use Cloudflare, this value should be greater than `0.01`.
+
+##### options.errorTtl
+
+Type: `number`<br>
+Default: `0.15`
+
+The time how long it needs to remember failed queries (TTL in seconds).
+
+**Note**: This option is independent, `options.maxTtl` does not affect this.
+
+**Pro Tip**: This shouldn't be lower than your DNS server response time in order to prevent bottlenecks. For example, if you use Cloudflare, this value should be greater than `0.01`.
+
 ##### options.resolver
@@ -128,3 +152,3 @@
 
-DNS servers used to make the query. Can be overridden - then the new servers will be used.
+The DNS servers used to make queries. Can be overridden - doing so will trigger `cacheableLookup.updateInterfaceInfo()`.
 
@@ -152,12 +176,2 @@ #### [lookup(hostname, options, callback)](https://nodejs.org/api/dns.html#dns_dns_lookup_hostname_options_callback)
 
-##### options.throwNotFound
-
-Type: `boolean`<br>
-Default: `true`
-
-If set to `false` and it gets no match, it will return `undefined`.
-If set to `true` and it gets no match, it will throw `ENOTFOUND` error.
-
-**Note**: This option is meant **only** for the asynchronous implementation! The callback version will always pass an error if no match found.
-
 #### query(hostname)
@@ -181,3 +195,3 @@
 
-Removes outdated entries.
+Removes outdated entries. It's automatically called on every lookup.
 
@@ -190,5 +204,5 @@ #### updateInterfaceInfo()
 
-#### clear()
+#### clear(hostname?)
 
-Clears the cache.
+Clears the cache for the given hostname. If the hostname argument is not present, the entire cache will be cleared.
 
@@ -203,14 +217,11 @@ ## High performance
 ```
-CacheableLookup#lookupAsync                x 2,024,888 ops/sec ±0.84%  (87 runs sampled)
-CacheableLookup#lookupAsync.all            x 2,093,860 ops/sec ±1.00%  (88 runs sampled)
-CacheableLookup#lookupAsync.all.ADDRCONFIG x 1,898,088 ops/sec ±0.61%  (89 runs sampled)
-CacheableLookup#lookup                     x 1,905,060 ops/sec ±0.76%  (90 runs sampled)
-CacheableLookup#lookup.all                 x 1,889,284 ops/sec ±1.37%  (87 runs sampled)
-CacheableLookup#lookup.all.ADDRCONFIG      x 1,740,616 ops/sec ±0.83%  (89 runs sampled)
-CacheableLookup#lookupAsync - zero TTL     x 226       ops/sec ±3.55%  (56 runs sampled)
-CacheableLookup#lookup      - zero TTL     x 228       ops/sec ±2.48%  (62 runs sampled)
-dns#resolve4                               x 346       ops/sec ±3.58%  (55 runs sampled)
-dns#lookup                                 x 20,368    ops/sec ±38.31% (53 runs sampled)
-dns#lookup.all                             x 13,529    ops/sec ±31.35% (29 runs sampled)
-dns#lookup.all.ADDRCONFIG                  x 6,211     ops/sec ±22.92% (26 runs sampled)
+CacheableLookup#lookupAsync                x 2,441,577 ops/sec ±0.57% (87 runs sampled)
+CacheableLookup#lookupAsync.all            x 2,539,120 ops/sec ±0.48% (88 runs sampled)
+CacheableLookup#lookupAsync.all.ADDRCONFIG x 2,228,416 ops/sec ±0.31% (88 runs sampled)
+CacheableLookup#lookup                     x 2,374,110 ops/sec ±0.29% (89 runs sampled)
+CacheableLookup#lookup.all                 x 2,311,587 ops/sec ±0.38% (88 runs sampled)
+CacheableLookup#lookup.all.ADDRCONFIG      x 2,074,475 ops/sec ±0.41% (90 runs sampled)
+dns#lookup                                 x 7,272     ops/sec ±0.36% (86 runs sampled)
+dns#lookup.all                             x 7,249     ops/sec ±0.40% (86 runs sampled)
+dns#lookup.all.ADDRCONFIG                  x 5,693     ops/sec ±0.28% (85 runs sampled)
 Fastest is CacheableLookup#lookupAsync.all
@@ -217,0 +228,0 @@ ```

source/hosts-resolver.js

 'use strict';
-const {stat, readFile} = require('fs').promises;
+const {watchFile} = require('fs');
+const {readFile} = require('fs').promises;
 const {isIP} = require('net');
 
 const isWindows = process.platform === 'win32';
-const hostsPath = isWindows ? 'C:\\Windows\\System32\\drivers\\etc\\hosts' : '/etc/hosts';
+const hostsPath = isWindows ? `${process.env.SystemDrive}\\Windows\\System32\\drivers\\etc\\hosts` : '/etc/hosts';
 
@@ -22,23 +23,29 @@ const hostnameRegExp = /^(?:(?:[a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9-]*[a-zA-Z0-9])\.)*(?:[A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9-]*[A-Za-z0-9])$/;
 		this._hostsPath = customHostsPath;
-		this._promise = undefined;
 		this._error = null;
 		this._hosts = {};
-		this._lastModifiedTime = 0;
 
-		this.update();
-	}
+		this._promise = (async () => {
+			if (typeof this._hostsPath !== 'string') {
+				return;
+			}
 
-	async _update() {
-		try {
-			const {_hostsPath} = this;
-			const {mtimeMs} = await stat(_hostsPath);
+			await this._update();
 
-			if (mtimeMs === this._lastModifiedTime) {
-				return this._hosts;
+			if (this._error) {
+				return;
 			}
 
-			this._lastModifiedTime = mtimeMs;
-			this._hosts = {};
+			watchFile(this._hostsPath, (currentTime, previousTime) => {
+				if (currentTime > previousTime) {
+					this._update();
+				}
+			});
 
-			let lines = await readFile(_hostsPath, fileOptions);
+			this._promise = null;
+		})();
+	}
+
+	async _update() {
+		try {
+			let lines = await readFile(this._hostsPath, fileOptions);
 			lines = lines.replace(whitespaceRegExp, ' ');
@@ -49,2 +56,4 @@ lines = lines.replace(tabRegExp, ' ');
 
+			this._hosts = {};
+
 			for (const line of lines) {
@@ -80,2 +89,3 @@ const parts = line.split(' ');
 						this._hosts[hostname] = [];
+						this._hosts[hostname].expires = Infinity;
 					}
@@ -97,16 +107,2 @@
 
-	async update() {
-		if (this._error || this._hostsPath === false) {
-			return this._hosts;
-		}
-
-		const promise = this._update();
-
-		this._promise = promise;
-		await promise;
-		this._promise = undefined;
-
-		return this._hosts;
-	}
-
 	async get(hostname) {
@@ -113,0 +109,0 @@ if (this._promise) {

source/index.js

 'use strict';
-const {V4MAPPED, ADDRCONFIG, promises: dnsPromises} = require('dns');
+const {
+	V4MAPPED,
+	ADDRCONFIG,
+	promises: {
+		Resolver: AsyncResolver
+	},
+	lookup
+} = require('dns');
 const {promisify} = require('util');
@@ -7,4 +14,2 @@ const os = require('os');
 
-const {Resolver: AsyncResolver} = dnsPromises;
-
 const kCacheableLookupCreateConnection = Symbol('cacheableLookupCreateConnection');
@@ -51,43 +56,2 @@ const kCacheableLookupInstance = Symbol('cacheableLookupInstance');
 
-class TTLMap {
-	constructor() {
-		this.values = new Map();
-		this.expiries = new Map();
-	}
-
-	set(key, value, ttl) {
-		this.values.set(key, value);
-		this.expiries.set(key, ttl && (ttl + Date.now()));
-	}
-
-	get(key) {
-		const expiry = this.expiries.get(key);
-
-		if (typeof expiry === 'number') {
-			if (Date.now() > expiry) {
-				this.values.delete(key);
-				this.expiries.delete(key);
-
-				return;
-			}
-		}
-
-		return this.values.get(key);
-	}
-
-	delete(key) {
-		this.values.delete(key);
-		return this.expiries.delete(key);
-	}
-
-	clear() {
-		this.values.clear();
-		this.expiries.clear();
-	}
-
-	get size() {
-		return this.values.size;
-	}
-}
-
 const ttl = {ttl: true};
@@ -97,12 +61,21 @@
 	constructor({
-		cache = new TTLMap(),
+		customHostsPath,
+		cache = new Map(),
 		maxTtl = Infinity,
 		resolver = new AsyncResolver(),
-		customHostsPath
+		fallbackTtl = 1,
+		errorTtl = 0.15
 	} = {}) {
 		this.maxTtl = maxTtl;
+		this.fallbackTtl = fallbackTtl;
+		this.errorTtl = errorTtl;
 
+		// This value is in milliseconds
+		this._lockTime = Math.max(Math.floor(Math.min(this.fallbackTtl * 1000, this.errorTtl * 1000)), 10);
+
 		this._cache = cache;
 		this._resolver = resolver;
 
+		this._lookup = promisify(lookup);
+
 		if (this._resolver instanceof AsyncResolver) {
@@ -120,2 +93,4 @@ this._resolve4 = this._resolver.resolve4.bind(this._resolver);
 
+		this._pending = {};
+
 		this.lookup = this.lookup.bind(this);
@@ -126,2 +101,4 @@ this.lookupAsync = this.lookupAsync.bind(this);
 	set servers(servers) {
+		this.updateInterfaceInfo();
+
 		this._resolver.setServers(servers);
@@ -138,6 +115,14 @@ }
 			options = {};
+		} else if (typeof options === 'number') {
+			options = {
+				family: options
+			};
 		}
 
+		if (!callback) {
+			throw new Error('Callback must be a function.');
+		}
+
 		// eslint-disable-next-line promise/prefer-await-to-then
-		this.lookupAsync(hostname, options, true).then(result => {
+		this.lookupAsync(hostname, options).then(result => {
 			if (options.all) {
@@ -148,6 +133,12 @@ callback(null, result);
 			}
-		}).catch(callback);
+		}, callback);
 	}
 
-	async lookupAsync(hostname, options = {}, throwNotFound = undefined) {
+	async lookupAsync(hostname, options = {}) {
+		if (typeof options === 'number') {
+			options = {
+				family: options
+			};
+		}
+
 		let cached = await this.query(hostname);
@@ -173,9 +164,7 @@
 		if (cached.length === 0) {
-			if (throwNotFound || options.throwNotFound !== false) {
-				const error = new Error(`ENOTFOUND ${hostname}`);
-				error.code = 'ENOTFOUND';
-				error.hostname = hostname;
+			const error = new Error(`ENOTFOUND ${hostname}`);
+			error.code = 'ENOTFOUND';
+			error.hostname = hostname;
 
-				throw error;
-			}
+			throw error;
 		}
@@ -191,10 +180,21 @@
 
-		return this._getEntry(cached);
+		return this._getEntry(cached, hostname);
 	}
 
 	async query(hostname) {
+		this.tick();
+
 		let cached = await this._hostsResolver.get(hostname) || await this._cache.get(hostname);
 
-		if (!cached || cached.length === 0) {
-			cached = await this.queryAndCache(hostname);
+		if (!cached) {
+			const pending = this._pending[hostname];
+
+			if (pending) {
+				cached = await pending;
+			} else {
+				const newPromise = this.queryAndCache(hostname);
+				this._pending[hostname] = newPromise;
+
+				cached = await newPromise;
+			}
 		}
@@ -210,6 +210,6 @@
 	async queryAndCache(hostname) {
+		// We could make an ANY query, but DNS servers may reject that.
 		const [As, AAAAs] = await Promise.all([this._resolve4(hostname, ttl).catch(() => []), this._resolve6(hostname, ttl).catch(() => [])]);
 
 		let cacheTtl = 0;
-		const now = Date.now();
 
@@ -219,4 +219,5 @@ if (As) {
 				entry.family = 4;
-				entry.expires = now + (entry.ttl * 1000);
+				entry.expires = Date.now() + (entry.ttl * 1000);
 
+				// Is the TTL the same for all entries?
 				cacheTtl = Math.max(cacheTtl, entry.ttl);
@@ -229,4 +230,5 @@ }
 				entry.family = 6;
-				entry.expires = now + (entry.ttl * 1000);
+				entry.expires = Date.now() + (entry.ttl * 1000);
 
+				// Is the TTL the same for all entries?
 				cacheTtl = Math.max(cacheTtl, entry.ttl);
@@ -236,14 +238,44 @@ }
 
-		const entries = [...(As || []), ...(AAAAs || [])];
+		let entries = [...(As || []), ...(AAAAs || [])];
 
-		cacheTtl = Math.min(this.maxTtl, cacheTtl) * 1000;
+		if (entries.length === 0) {
+			try {
+				entries = await this._lookup(hostname, {
+					all: true
+				});
 
+				for (const entry of entries) {
+					entry.ttl = this.fallbackTtl;
+					entry.expires = Date.now() + (entry.ttl * 1000);
+				}
+
+				cacheTtl = this.fallbackTtl * 1000;
+			} catch (error) {
+				delete this._pending[hostname];
+
+				if (error.code === 'ENOTFOUND') {
+					cacheTtl = this.errorTtl * 1000;
+
+					entries.expires = Date.now() + cacheTtl;
+					await this._cache.set(hostname, entries, cacheTtl);
+				}
+
+				throw error;
+			}
+		} else {
+			cacheTtl = Math.min(this.maxTtl, cacheTtl) * 1000;
+		}
+
 		if (this.maxTtl > 0 && cacheTtl > 0) {
+			entries.expires = Date.now() + cacheTtl;
 			await this._cache.set(hostname, entries, cacheTtl);
 		}
 
+		delete this._pending[hostname];
+
 		return entries;
 	}
 
-	_getEntry(entries) {
+	// eslint-disable-next-line no-unused-vars
+	_getEntry(entries, hostname) {
 		return entries[Math.floor(Math.random() * entries.length)];
@@ -257,7 +289,7 @@ }
 
-		if (this._cache instanceof TTLMap) {
+		if (this._cache instanceof Map) {
 			const now = Date.now();
 
-			for (const [hostname, expiry] of this._cache.expiries) {
-				if (now > expiry) {
+			for (const [hostname, {expires}] of this._cache) {
+				if (now >= expires) {
 					this._cache.delete(hostname);
@@ -268,4 +300,2 @@ }
 
-		this._hostsResolver.update();
-
 		this._tickLocked = true;
@@ -275,3 +305,3 @@
 			this._tickLocked = false;
-		}, 1000).unref();
+		}, this._lockTime).unref();
 	}
@@ -292,5 +322,2 @@
 				options.lookup = this.lookup;
-
-				// Make sure the database is up to date
-				this.tick();
 			}
@@ -319,7 +346,11 @@
 		this._iface = getIfaceInfo();
-		this._hostsResolver.update();
 		this._cache.clear();
 	}
 
-	clear() {
+	clear(hostname) {
+		if (hostname) {
+			this._cache.delete(hostname);
+			return;
+		}
+
 		this._cache.clear();
@@ -326,0 +357,0 @@ }

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Environment variable access

Supply chain risk

Package accesses environment variables, which may be a sign of credential stuffing or data theft.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

24515

increased by7.9%

Lines of code

495

increased by8.08%

Number of lines in readme file

236

increased by4.89%

Worsened metrics

Number of low supply chain risk alerts

3

increased by200%

No dependency changes