tardis-dev - npm Package Compare versions

Comparing version 10.1.15 to 10.1.16

dist/combine.js

@@ -21,5 +21,14 @@ "use strict";
     }
-    if (current.result.value.localTimestamp < oldest.result.value.localTimestamp) {
+    const currentTimestamp = current.result.value.localTimestamp.valueOf();
+    const oldestTimestamp = oldest.result.value.localTimestamp.valueOf();
+    if (currentTimestamp < oldestTimestamp) {
         return current;
     }
+    if (currentTimestamp === oldestTimestamp) {
+        const currentTimestampMicroSeconds = current.result.value.localTimestamp.μs || 0;
+        const oldestTimestampMicroSeconds = oldest.result.value.localTimestamp.μs || 0;
+        if (currentTimestampMicroSeconds < oldestTimestampMicroSeconds) {
+            return current;
+        }
+    }
     return oldest;
@@ -26,0 +35,0 @@ }

dist/computable/booksnapshot.d.ts

@@ -1,4 +0,4 @@
-import { BookChange, BookPriceLevel, BookSnapshot } from '../types';
+import { OnLevelRemovedCB } from '../orderbook';
+import { BookSnapshot } from '../types';
 import { Computable } from './computable';
-declare type OnLevelRemovedCB = (bookChange: BookChange, bestBidBeforeRemoval: BookPriceLevel | undefined, bestBidAfterRemoval: BookPriceLevel | undefined, bestAskBeforeRemoval: BookPriceLevel | undefined, bestAskAfterRemoval: BookPriceLevel | undefined) => void;
 declare type BookSnapshotComputableOptions = {
@@ -5,0 +5,0 @@ name?: string;

dist/computable/booksnapshot.js

@@ -24,3 +24,2 @@ "use strict";
         this._type = 'book_snapshot';
-        this._orderBook = new orderbook_1.OrderBook();
         this._lastUpdateTimestamp = new Date(-1);
@@ -31,4 +30,6 @@ this._bids = [];
         this._interval = interval;
-        this._removeCrossedLevels = removeCrossedLevels;
-        this._onCrossedLevelRemoved = onCrossedLevelRemoved;
+        this._orderBook = new orderbook_1.OrderBook({
+            removeCrossedLevels,
+            onCrossedLevelRemoved
+        });
         // initialize all bids/asks levels to empty ones
@@ -77,35 +78,2 @@ for (let i = 0; i < this._depth; i++) {
         const asksIterable = this._orderBook.asks();
-        if (this._removeCrossedLevels) {
-            let bestBid = this._orderBook.bestBid();
-            let bestAsk = this._orderBook.bestAsk();
-            let bookIsCrossed = bestBid !== undefined && bestAsk !== undefined && bestBid.price >= bestAsk.price;
-            // if after update we have crossed order book (best bid >= best ask)
-            // it most likely means that exchange has not published delete message for the other side of the book
-            // more info:
-            // https://www.reddit.com/r/KrakenSupport/comments/d1a4nx/websocket_orderbook_receiving_wrong_bid_price_for/
-            // https://www.reddit.com/r/BitMEX/comments/8lbj9e/bidask_ledger_weirdness/
-            // https://twitter.com/coinarb/status/931260529993170944
-            if (bookIsCrossed) {
-                // decide from which side of the book we should remove level so book isn't crossed anymore
-                // if current book update updated "best ask" it means we should remove "best bid" as exchange hasn't provided book change update
-                // that deletes it, and vice versa for for "best bids"
-                const shouldRemoveBestBid = bookChange.asks.some((s) => s.price === bestAsk.price);
-                while (bookIsCrossed) {
-                    if (shouldRemoveBestBid) {
-                        this._orderBook.removeBestBid();
-                    }
-                    else {
-                        this._orderBook.removeBestAsk();
-                    }
-                    const newBestBid = this._orderBook.bestBid();
-                    const newBestAsk = this._orderBook.bestAsk();
-                    if (this._onCrossedLevelRemoved !== undefined) {
-                        this._onCrossedLevelRemoved(bookChange, bestBid, newBestBid, bestAsk, newBestAsk);
-                    }
-                    bestBid = newBestBid;
-                    bestAsk = newBestAsk;
-                    bookIsCrossed = bestBid !== undefined && bestAsk !== undefined && bestBid.price >= bestAsk.price;
-                }
-            }
-        }
         for (let i = 0; i < this._depth; i++) {
@@ -112,0 +80,0 @@ const bidLevelResult = bidsIterable.next();

dist/consts.d.ts

@@ -1,2 +0,2 @@
-export declare const EXCHANGES: readonly ["bitmex", "deribit", "binance-futures", "binance-delivery", "binance", "ftx", "okex-futures", "okex-options", "okex-swap", "okex", "huobi-dm", "huobi-dm-swap", "huobi", "bitfinex-derivatives", "bitfinex", "bitfinex-alts", "cryptofacilities", "kraken", "bitstamp", "coinbase", "gemini", "coinflex", "bybit", "phemex", "ftx-us", "okcoin", "bitflyer", "hitbtc", "delta", "binance-jersey", "binance-us", "binance-dex"];
+export declare const EXCHANGES: readonly ["bitmex", "deribit", "binance-futures", "binance-delivery", "binance", "ftx", "okex-futures", "okex-options", "okex-swap", "okex", "huobi-dm", "huobi-dm-swap", "huobi", "bitfinex-derivatives", "bitfinex", "bitfinex-alts", "cryptofacilities", "kraken", "bitstamp", "coinbase", "gemini", "coinflex", "bybit", "phemex", "ftx-us", "gate-io", "okcoin", "bitflyer", "hitbtc", "delta", "binance-jersey", "binance-us", "binance-dex"];
 export declare const EXCHANGE_CHANNELS_INFO: {
@@ -26,3 +26,3 @@ bitmex: readonly ["trade", "orderBookL2", "liquidation", "connected", "announcement", "chat", "publicNotifications", "instrument", "settlement", "funding", "insurance", "orderBookL2_25", "quote", "quoteBin1m", "quoteBin5m", "quoteBin1h", "quoteBin1d", "tradeBin1m", "tradeBin5m", "tradeBin1h", "tradeBin1d"];
     'bitfinex-derivatives': readonly ["trades", "book", "raw_book", "status", "liquidations"];
-    huobi: readonly ["depth", "detail", "trade", "bbo"];
+    huobi: readonly ["depth", "detail", "trade", "bbo", "mbp"];
     'huobi-dm': readonly ["depth", "detail", "trade", "bbo", "basis", "liquidation_orders", "contract_info", "open_interest"];
@@ -36,3 +36,4 @@ 'huobi-dm-swap': readonly ["depth", "detail", "trade", "basis", "funding_rate", "liquidation_orders", "contract_info", "open_interest"];
     delta: string[];
+    'gate-io': string[];
 };
 //# sourceMappingURL=consts.d.ts.map

dist/consts.js

@@ -30,2 +30,3 @@ "use strict";
     'ftx-us',
+    'gate-io',
     'okcoin',
@@ -151,3 +152,3 @@ 'bitflyer',
 const BITFINEX_DERIV_CHANNELS = ['trades', 'book', 'raw_book', 'status', 'liquidations'];
-const HUOBI_CHANNELS = ['depth', 'detail', 'trade', 'bbo'];
+const HUOBI_CHANNELS = ['depth', 'detail', 'trade', 'bbo', 'mbp'];
 const HUOBI_DM_CHANNELS = ['depth', 'detail', 'trade', 'bbo', 'basis', 'liquidation_orders', 'contract_info', 'open_interest'];
@@ -179,2 +180,3 @@ const HUOBI_DM_SWAP_CHANNELS = [
 ];
+const GATE_IO_CHANNELS = ['trades', 'depth', 'ticker'];
 exports.EXCHANGE_CHANNELS_INFO = {
@@ -212,4 +214,5 @@ bitmex: BITMEX_CHANNELS,
     phemex: PHEMEX_CHANNELS,
-    delta: DELTA_CHANNELS
+    delta: DELTA_CHANNELS,
+    'gate-io': GATE_IO_CHANNELS,
 };
 //# sourceMappingURL=consts.js.map

dist/handy.d.ts

@@ -33,2 +33,21 @@ import { Mapper } from './mappers';
 }): Promise<void>;
+export declare class CircularBuffer<T> {
+    private readonly _bufferSize;
+    private _buffer;
+    private _index;
+    constructor(_bufferSize: number);
+    append(value: T): T | undefined;
+    items(): Generator<T, void, unknown>;
+    get count(): number;
+    clear(): void;
+}
+export declare class CappedSet<T> {
+    private readonly _maxSize;
+    private _set;
+    constructor(_maxSize: number);
+    has(value: T): boolean;
+    add(value: T): void;
+    remove(value: T): void;
+    size(): number;
+}
 //# sourceMappingURL=handy.d.ts.map

dist/handy.js

@@ -25,3 +25,3 @@ "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.download = exports.optimizeFilters = exports.parseμs = exports.batch = exports.getFilters = exports.normalizeMessages = exports.take = exports.HttpError = exports.ONE_SEC_IN_MS = exports.sequence = exports.addDays = exports.addMinutes = exports.sha256 = exports.doubleDigit = exports.formatDateToPath = exports.wait = exports.parseAsUTCDate = void 0;
+exports.CappedSet = exports.CircularBuffer = exports.download = exports.optimizeFilters = exports.parseμs = exports.batch = exports.getFilters = exports.normalizeMessages = exports.take = exports.HttpError = exports.ONE_SEC_IN_MS = exports.sequence = exports.addDays = exports.addMinutes = exports.sha256 = exports.doubleDigit = exports.formatDateToPath = exports.wait = exports.parseAsUTCDate = void 0;
 const crypto_1 = __importStar(require("crypto"));
@@ -227,3 +227,4 @@ const fs_extra_1 = require("fs-extra");
     keepAlive: true,
-    keepAliveMsecs: 10 * exports.ONE_SEC_IN_MS
+    keepAliveMsecs: 10 * exports.ONE_SEC_IN_MS,
+    maxSockets: 120
 });
@@ -233,3 +234,3 @@ async function download({ apiKey, downloadPath, url, userAgent }) {
         agent: httpsAgent,
-        timeout: 45 * exports.ONE_SEC_IN_MS,
+        timeout: 90 * exports.ONE_SEC_IN_MS,
         headers: {
@@ -318,2 +319,55 @@ 'Accept-Encoding': 'gzip',
 }
+class CircularBuffer {
+    constructor(_bufferSize) {
+        this._bufferSize = _bufferSize;
+        this._buffer = [];
+        this._index = 0;
+    }
+    append(value) {
+        const isFull = this._buffer.length === this._bufferSize;
+        let poppedValue;
+        if (isFull) {
+            poppedValue = this._buffer[this._index];
+        }
+        this._buffer[this._index] = value;
+        this._index = (this._index + 1) % this._bufferSize;
+        return poppedValue;
+    }
+    *items() {
+        for (let i = 0; i < this._buffer.length; i++) {
+            const index = (this._index + i) % this._buffer.length;
+            yield this._buffer[index];
+        }
+    }
+    get count() {
+        return this._buffer.length;
+    }
+    clear() {
+        this._buffer = [];
+        this._index = 0;
+    }
+}
+exports.CircularBuffer = CircularBuffer;
+class CappedSet {
+    constructor(_maxSize) {
+        this._maxSize = _maxSize;
+        this._set = new Set();
+    }
+    has(value) {
+        return this._set.has(value);
+    }
+    add(value) {
+        if (this._set.size >= this._maxSize) {
+            this._set.delete(this._set.keys().next().value);
+        }
+        this._set.add(value);
+    }
+    remove(value) {
+        this._set.delete(value);
+    }
+    size() {
+        return this._set.size;
+    }
+}
+exports.CappedSet = CappedSet;
 //# sourceMappingURL=handy.js.map

dist/index.d.ts

@@ -15,2 +15,3 @@ export * from './apikeyaccessinfo';
 export * from './types';
+export * from './filter';
 //# sourceMappingURL=index.d.ts.map

dist/index.js

@@ -28,2 +28,3 @@ "use strict";
 __exportStar(require("./types"), exports);
+__exportStar(require("./filter"), exports);
 //# sourceMappingURL=index.js.map

dist/mappers/binance.d.ts

@@ -0,1 +1,2 @@
+import { CircularBuffer } from '../handy';
 import { BookChange, DerivativeTicker, Exchange, FilterForExchange, Trade } from '../types';
@@ -13,12 +14,2 @@ import { Mapper } from './mapper';
 }
-declare class CircularBuffer<T> {
-    private readonly _bufferSize;
-    private _buffer;
-    private _index;
-    constructor(_bufferSize: number);
-    append(value: T): T | undefined;
-    items(): Generator<T, void, unknown>;
-    get count(): number;
-    clear(): void;
-}
 export declare class BinanceBookChangeMapper implements Mapper<'binance' | 'binance-jersey' | 'binance-us' | 'binance-futures' | 'binance-delivery', BookChange> {
@@ -25,0 +16,0 @@ protected readonly exchange: Exchange;

dist/mappers/binance.js

@@ -5,2 +5,3 @@ "use strict";
 const debug_1 = require("../debug");
+const handy_1 = require("../handy");
 const mapper_1 = require("./mapper");
@@ -44,32 +45,2 @@ // https://github.com/binance-exchange/binance-official-api-docs/blob/master/web-socket-streams.md
 exports.BinanceTradesMapper = BinanceTradesMapper;
-class CircularBuffer {
-    constructor(_bufferSize) {
-        this._bufferSize = _bufferSize;
-        this._buffer = [];
-        this._index = 0;
-    }
-    append(value) {
-        const isFull = this._buffer.length === this._bufferSize;
-        let poppedValue;
-        if (isFull) {
-            poppedValue = this._buffer[this._index];
-        }
-        this._buffer[this._index] = value;
-        this._index = (this._index + 1) % this._bufferSize;
-        return poppedValue;
-    }
-    *items() {
-        for (let i = 0; i < this._buffer.length; i++) {
-            const index = (this._index + i) % this._buffer.length;
-            yield this._buffer[index];
-        }
-    }
-    get count() {
-        return this._buffer.length;
-    }
-    clear() {
-        this._buffer = [];
-        this._index = 0;
-    }
-}
 class BinanceBookChangeMapper {
@@ -104,3 +75,3 @@ constructor(exchange, ignoreBookSnapshotOverlapError) {
             this.symbolToDepthInfoMapping[symbol] = {
-                bufferedUpdates: new CircularBuffer(200)
+                bufferedUpdates: new handy_1.CircularBuffer(200)
             };
@@ -107,0 +78,0 @@ }

dist/mappers/bybit.d.ts

@@ -29,3 +29,3 @@ import { BookChange, DerivativeTicker, Exchange, Trade } from '../types';
         readonly symbol: string;
-        readonly exchange: "bitmex" | "deribit" | "binance-futures" | "binance-delivery" | "binance" | "ftx" | "okex-futures" | "okex-options" | "okex-swap" | "okex" | "huobi-dm" | "huobi-dm-swap" | "huobi" | "bitfinex-derivatives" | "bitfinex" | "bitfinex-alts" | "cryptofacilities" | "kraken" | "bitstamp" | "coinbase" | "gemini" | "coinflex" | "bybit" | "phemex" | "ftx-us" | "okcoin" | "bitflyer" | "hitbtc" | "delta" | "binance-jersey" | "binance-us" | "binance-dex";
+        readonly exchange: "bitmex" | "deribit" | "binance-futures" | "binance-delivery" | "binance" | "ftx" | "okex-futures" | "okex-options" | "okex-swap" | "okex" | "huobi-dm" | "huobi-dm-swap" | "huobi" | "bitfinex-derivatives" | "bitfinex" | "bitfinex-alts" | "cryptofacilities" | "kraken" | "bitstamp" | "coinbase" | "gemini" | "coinflex" | "bybit" | "phemex" | "ftx-us" | "gate-io" | "okcoin" | "bitflyer" | "hitbtc" | "delta" | "binance-jersey" | "binance-us" | "binance-dex";
         readonly isSnapshot: boolean;
@@ -32,0 +32,0 @@ readonly bids: {

dist/mappers/coinbase.js

 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.CoinbaseBookChangMapper = exports.coinbaseTradesMapper = void 0;
+const handy_1 = require("../handy");
 // https://docs.pro.coinbase.com/#websocket-feed
@@ -18,2 +19,4 @@ exports.coinbaseTradesMapper = {
     *map(message, localTimestamp) {
+        const timestamp = new Date(message.time);
+        timestamp.μs = handy_1.parseμs(message.time);
         yield {
@@ -27,3 +30,3 @@ type: 'trade',
             side: message.side === 'sell' ? 'buy' : 'sell',
-            timestamp: new Date(message.time),
+            timestamp,
             localTimestamp: localTimestamp
@@ -87,2 +90,3 @@ };
             else {
+                timestamp.μs = handy_1.parseμs(message.time);
                 this._symbolLastTimestampMap.set(message.product_id, timestamp);
@@ -89,0 +93,0 @@ }

dist/mappers/huobi.d.ts

 import { BookChange, DerivativeTicker, Exchange, Trade } from '../types';
 import { Mapper } from './mapper';
+import { CircularBuffer } from '../handy';
 export declare class HuobiTradesMapper implements Mapper<'huobi' | 'huobi-dm' | 'huobi-dm-swap', Trade> {
@@ -25,3 +26,3 @@ private readonly _exchange;
         readonly symbol: string;
-        readonly exchange: "bitmex" | "deribit" | "binance-futures" | "binance-delivery" | "binance" | "ftx" | "okex-futures" | "okex-options" | "okex-swap" | "okex" | "huobi-dm" | "huobi-dm-swap" | "huobi" | "bitfinex-derivatives" | "bitfinex" | "bitfinex-alts" | "cryptofacilities" | "kraken" | "bitstamp" | "coinbase" | "gemini" | "coinflex" | "bybit" | "phemex" | "ftx-us" | "okcoin" | "bitflyer" | "hitbtc" | "delta" | "binance-jersey" | "binance-us" | "binance-dex";
+        readonly exchange: "bitmex" | "deribit" | "binance-futures" | "binance-delivery" | "binance" | "ftx" | "okex-futures" | "okex-options" | "okex-swap" | "okex" | "huobi-dm" | "huobi-dm-swap" | "huobi" | "bitfinex-derivatives" | "bitfinex" | "bitfinex-alts" | "cryptofacilities" | "kraken" | "bitstamp" | "coinbase" | "gemini" | "coinflex" | "bybit" | "phemex" | "ftx-us" | "gate-io" | "okcoin" | "bitflyer" | "hitbtc" | "delta" | "binance-jersey" | "binance-us" | "binance-dex";
         readonly isSnapshot: boolean;
@@ -41,2 +42,47 @@ readonly bids: {
 }
+export declare class HuobiMBPBookChangeMapper implements Mapper<'huobi', BookChange> {
+    protected readonly _exchange: Exchange;
+    protected readonly symbolToMBPInfoMapping: {
+        [key: string]: MBPInfo;
+    };
+    constructor(_exchange: Exchange);
+    canHandle(message: any): any;
+    getFilters(symbols?: string[]): {
+        readonly channel: "mbp";
+        readonly symbols: string[] | undefined;
+    }[];
+    map(message: HuobiMBPDataMessage | HuobiMBPSnapshot, localTimestamp: Date): Generator<{
+        readonly type: "book_change";
+        readonly symbol: string;
+        readonly exchange: "bitmex" | "deribit" | "binance-futures" | "binance-delivery" | "binance" | "ftx" | "okex-futures" | "okex-options" | "okex-swap" | "okex" | "huobi-dm" | "huobi-dm-swap" | "huobi" | "bitfinex-derivatives" | "bitfinex" | "bitfinex-alts" | "cryptofacilities" | "kraken" | "bitstamp" | "coinbase" | "gemini" | "coinflex" | "bybit" | "phemex" | "ftx-us" | "gate-io" | "okcoin" | "bitflyer" | "hitbtc" | "delta" | "binance-jersey" | "binance-us" | "binance-dex";
+        readonly isSnapshot: false;
+        readonly bids: {
+            price: number;
+            amount: number;
+        }[];
+        readonly asks: {
+            price: number;
+            amount: number;
+        }[];
+        readonly timestamp: Date;
+        readonly localTimestamp: Date;
+    } | {
+        readonly type: "book_change";
+        readonly symbol: string;
+        readonly exchange: "bitmex" | "deribit" | "binance-futures" | "binance-delivery" | "binance" | "ftx" | "okex-futures" | "okex-options" | "okex-swap" | "okex" | "huobi-dm" | "huobi-dm-swap" | "huobi" | "bitfinex-derivatives" | "bitfinex" | "bitfinex-alts" | "cryptofacilities" | "kraken" | "bitstamp" | "coinbase" | "gemini" | "coinflex" | "bybit" | "phemex" | "ftx-us" | "gate-io" | "okcoin" | "bitflyer" | "hitbtc" | "delta" | "binance-jersey" | "binance-us" | "binance-dex";
+        readonly isSnapshot: true;
+        readonly bids: {
+            price: number;
+            amount: number;
+        }[];
+        readonly asks: {
+            price: number;
+            amount: number;
+        }[];
+        readonly timestamp: Date;
+        readonly localTimestamp: Date;
+    }, void, unknown>;
+    private _mapMBPUpdate;
+    private _mapBookLevel;
+}
 export declare class HuobiDerivativeTickerMapper implements Mapper<'huobi-dm' | 'huobi-dm-swap', DerivativeTicker> {
@@ -76,4 +122,4 @@ private readonly _exchange;
     tick: {
-        bids: HuobiBookLevel[] | null;
-        asks: HuobiBookLevel[] | null;
+        bids?: HuobiBookLevel[] | null;
+        asks?: HuobiBookLevel[] | null;
         event: 'snapshot' | 'update';
@@ -106,3 +152,25 @@ };
 };
+declare type HuobiMBPDataMessage = HuobiDataMessage & {
+    ts: number;
+    tick: {
+        bids?: HuobiBookLevel[] | null;
+        asks?: HuobiBookLevel[] | null;
+        seqNum: number;
+        prevSeqNum: number;
+    };
+};
+declare type HuobiMBPSnapshot = {
+    ts: number;
+    rep: string;
+    data: {
+        bids: HuobiBookLevel[];
+        asks: HuobiBookLevel[];
+        seqNum: number;
+    };
+};
+declare type MBPInfo = {
+    bufferedUpdates: CircularBuffer<HuobiMBPDataMessage>;
+    snapshotProcessed?: boolean;
+};
 export {};
 //# sourceMappingURL=huobi.d.ts.map

dist/mappers/huobi.js

 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.HuobiDerivativeTickerMapper = exports.HuobiBookChangeMapper = exports.HuobiTradesMapper = void 0;
+exports.HuobiDerivativeTickerMapper = exports.HuobiMBPBookChangeMapper = exports.HuobiBookChangeMapper = exports.HuobiTradesMapper = void 0;
 const mapper_1 = require("./mapper");
+const handy_1 = require("../handy");
 // https://huobiapi.github.io/docs/spot/v1/en/#websocket-market-data
@@ -75,2 +76,5 @@ // https://github.com/huobiapi/API_Docs_en/wiki/WS_api_reference_en
         const asks = Array.isArray(data.asks) ? data.asks : [];
+        if (bids.length === 0 && asks.length === 0) {
+            return;
+        }
         yield {
@@ -92,2 +96,116 @@ type: 'book_change',
 exports.HuobiBookChangeMapper = HuobiBookChangeMapper;
+function isSnapshot(message) {
+    return 'rep' in message;
+}
+class HuobiMBPBookChangeMapper {
+    constructor(_exchange) {
+        this._exchange = _exchange;
+        this.symbolToMBPInfoMapping = {};
+    }
+    canHandle(message) {
+        const channel = message.ch || message.rep;
+        if (channel === undefined) {
+            return false;
+        }
+        return channel.includes('.mbp.');
+    }
+    getFilters(symbols) {
+        symbols = normalizeSymbols(symbols);
+        return [
+            {
+                channel: 'mbp',
+                symbols
+            }
+        ];
+    }
+    *map(message, localTimestamp) {
+        const symbol = (isSnapshot(message) ? message.rep : message.ch).split('.')[1].toUpperCase();
+        if (this.symbolToMBPInfoMapping[symbol] === undefined) {
+            this.symbolToMBPInfoMapping[symbol] = {
+                bufferedUpdates: new handy_1.CircularBuffer(200)
+            };
+        }
+        const mbpInfo = this.symbolToMBPInfoMapping[symbol];
+        const snapshotAlreadyProcessed = mbpInfo.snapshotProcessed;
+        if (isSnapshot(message)) {
+            if (snapshotAlreadyProcessed) {
+                return;
+            }
+            const snapshotBids = message.data.bids.map(this._mapBookLevel);
+            const snapshotAsks = message.data.asks.map(this._mapBookLevel);
+            // if there were any depth updates buffered, let's proccess those by adding to or updating the initial snapshot
+            // when prevSeqNum >= snapshot seqNum
+            for (const update of mbpInfo.bufferedUpdates.items()) {
+                if (update.tick.prevSeqNum < message.data.seqNum) {
+                    continue;
+                }
+                const bookChange = this._mapMBPUpdate(update, symbol, localTimestamp);
+                if (bookChange !== undefined) {
+                    for (const bid of bookChange.bids) {
+                        const matchingBid = snapshotBids.find((b) => b.price === bid.price);
+                        if (matchingBid !== undefined) {
+                            matchingBid.amount = bid.amount;
+                        }
+                        else {
+                            snapshotBids.push(bid);
+                        }
+                    }
+                    for (const ask of bookChange.asks) {
+                        const matchingAsk = snapshotAsks.find((a) => a.price === ask.price);
+                        if (matchingAsk !== undefined) {
+                            matchingAsk.amount = ask.amount;
+                        }
+                        else {
+                            snapshotAsks.push(ask);
+                        }
+                    }
+                }
+            }
+            mbpInfo.snapshotProcessed = true;
+            mbpInfo.bufferedUpdates.clear();
+            yield {
+                type: 'book_change',
+                symbol,
+                exchange: this._exchange,
+                isSnapshot: true,
+                bids: snapshotBids,
+                asks: snapshotAsks,
+                timestamp: new Date(message.ts),
+                localTimestamp
+            };
+        }
+        else if (snapshotAlreadyProcessed) {
+            // snapshot was already processed let's map the mbp message as normal book_change
+            const update = this._mapMBPUpdate(message, symbol, localTimestamp);
+            if (update !== undefined) {
+                yield update;
+            }
+        }
+        else {
+            // there was no snapshot yet, let's buffer the update
+            mbpInfo.bufferedUpdates.append(message);
+        }
+    }
+    _mapMBPUpdate(message, symbol, localTimestamp) {
+        const bids = Array.isArray(message.tick.bids) ? message.tick.bids : [];
+        const asks = Array.isArray(message.tick.asks) ? message.tick.asks : [];
+        if (bids.length === 0 && asks.length === 0) {
+            return;
+        }
+        return {
+            type: 'book_change',
+            symbol,
+            exchange: this._exchange,
+            isSnapshot: false,
+            bids: bids.map(this._mapBookLevel),
+            asks: asks.map(this._mapBookLevel),
+            timestamp: new Date(message.ts),
+            localTimestamp: localTimestamp
+        };
+    }
+    _mapBookLevel(level) {
+        return { price: level[0], amount: level[1] };
+    }
+}
+exports.HuobiMBPBookChangeMapper = HuobiMBPBookChangeMapper;
 function normalizeSymbols(symbols) {
@@ -94,0 +212,0 @@ if (symbols !== undefined) {

dist/mappers/index.d.ts

 import { BookChange, DerivativeTicker, OptionSummary, Trade } from '../types';
 import { Mapper } from './mapper';
 export * from './mapper';
-export declare const normalizeTrades: <T extends "bitmex" | "deribit" | "binance-futures" | "binance-delivery" | "binance" | "ftx" | "okex-futures" | "okex-options" | "okex-swap" | "okex" | "huobi-dm" | "huobi-dm-swap" | "huobi" | "bitfinex-derivatives" | "bitfinex" | "bitfinex-alts" | "cryptofacilities" | "kraken" | "bitstamp" | "coinbase" | "gemini" | "bybit" | "phemex" | "ftx-us" | "okcoin" | "bitflyer" | "hitbtc" | "delta" | "binance-jersey" | "binance-us" | "binance-dex">(exchange: T, _localTimestamp: Date) => Mapper<T, Trade>;
-export declare const normalizeBookChanges: <T extends "bitmex" | "deribit" | "binance-futures" | "binance-delivery" | "binance" | "ftx" | "okex-futures" | "okex-options" | "okex-swap" | "okex" | "huobi-dm" | "huobi-dm-swap" | "huobi" | "bitfinex-derivatives" | "bitfinex" | "bitfinex-alts" | "cryptofacilities" | "kraken" | "bitstamp" | "coinbase" | "gemini" | "bybit" | "phemex" | "ftx-us" | "okcoin" | "bitflyer" | "hitbtc" | "delta" | "binance-jersey" | "binance-us" | "binance-dex">(exchange: T, localTimestamp: Date) => Mapper<T, BookChange>;
+export declare const normalizeTrades: <T extends "bitmex" | "deribit" | "binance-futures" | "binance-delivery" | "binance" | "ftx" | "okex-futures" | "okex-options" | "okex-swap" | "okex" | "huobi-dm" | "huobi-dm-swap" | "huobi" | "bitfinex-derivatives" | "bitfinex" | "bitfinex-alts" | "cryptofacilities" | "kraken" | "bitstamp" | "coinbase" | "gemini" | "bybit" | "phemex" | "ftx-us" | "gate-io" | "okcoin" | "bitflyer" | "hitbtc" | "delta" | "binance-jersey" | "binance-us" | "binance-dex">(exchange: T, _localTimestamp: Date) => Mapper<T, Trade>;
+export declare const normalizeBookChanges: <T extends "bitmex" | "deribit" | "binance-futures" | "binance-delivery" | "binance" | "ftx" | "okex-futures" | "okex-options" | "okex-swap" | "okex" | "huobi-dm" | "huobi-dm-swap" | "huobi" | "bitfinex-derivatives" | "bitfinex" | "bitfinex-alts" | "cryptofacilities" | "kraken" | "bitstamp" | "coinbase" | "gemini" | "bybit" | "phemex" | "ftx-us" | "gate-io" | "okcoin" | "bitflyer" | "hitbtc" | "delta" | "binance-jersey" | "binance-us" | "binance-dex">(exchange: T, localTimestamp: Date) => Mapper<T, BookChange>;
 export declare const normalizeDerivativeTickers: <T extends "bitmex" | "deribit" | "binance-futures" | "binance-delivery" | "ftx" | "okex-futures" | "okex-swap" | "huobi-dm" | "huobi-dm-swap" | "bitfinex-derivatives" | "cryptofacilities" | "bybit" | "phemex" | "delta">(exchange: T, _localTimestamp: Date) => Mapper<T, DerivativeTicker>;
 export declare const normalizeOptionsSummary: <T extends "deribit" | "okex-options">(exchange: T, _localTimestamp: Date) => Mapper<T, OptionSummary>;
 //# sourceMappingURL=index.d.ts.map

dist/mappers/index.js

@@ -33,2 +33,3 @@ "use strict";
 const phemex_1 = require("./phemex");
+const gateio_1 = require("./gateio");
 __exportStar(require("./mapper"), exports);
@@ -73,3 +74,4 @@ const THREE_MINUTES_IN_MS = 3 * 60 * handy_1.ONE_SEC_IN_MS;
     phemex: () => phemex_1.phemexTradesMapper,
-    delta: () => delta_1.deltaTradesMapper
+    delta: () => delta_1.deltaTradesMapper,
+    'gate-io': () => new gateio_1.GateIOTradesMapper('gate-io')
 };
@@ -100,3 +102,5 @@ const bookChangeMappers = {
     'okex-options': (localTimestamp) => new okex_1.OkexBookChangeMapper('okex-options', 'option', localTimestamp.valueOf() >= new Date('2020-02-08').valueOf()),
-    huobi: () => new huobi_1.HuobiBookChangeMapper('huobi'),
+    huobi: (localTimestamp) => localTimestamp.valueOf() >= new Date('2020-07-03').valueOf()
+        ? new huobi_1.HuobiMBPBookChangeMapper('huobi')
+        : new huobi_1.HuobiBookChangeMapper('huobi'),
     'huobi-dm': () => new huobi_1.HuobiBookChangeMapper('huobi-dm'),
@@ -108,3 +112,4 @@ 'huobi-dm-swap': () => new huobi_1.HuobiBookChangeMapper('huobi-dm-swap'),
     phemex: () => phemex_1.phemexBookChangeMapper,
-    delta: () => delta_1.deltaBookChangeMapper
+    delta: () => delta_1.deltaBookChangeMapper,
+    'gate-io': () => new gateio_1.GateBookChangeMapper('gate-io')
 };
@@ -111,0 +116,0 @@ const derivativeTickersMappers = {

dist/mappers/okex.js

@@ -86,2 +86,5 @@ "use strict";
         for (const message of okexDepthDataMessage.data) {
+            if (message.bids.length === 0 && message.asks.length === 0) {
+                continue;
+            }
             yield {
@@ -88,0 +91,0 @@ type: 'book_change',

dist/orderbook.d.ts

 import { BookChange, BookPriceLevel } from './types';
+export declare type OnLevelRemovedCB = (bookChange: BookChange, bestBidBeforeRemoval: BookPriceLevel | undefined, bestBidAfterRemoval: BookPriceLevel | undefined, bestAskBeforeRemoval: BookPriceLevel | undefined, bestAskAfterRemoval: BookPriceLevel | undefined) => void;
 export declare class OrderBook {
     private readonly _bids;
     private readonly _asks;
+    private readonly _removeCrossedLevels;
+    private readonly _onCrossedLevelRemoved;
     private _receivedInitialSnapshot;
+    constructor({ removeCrossedLevels, onCrossedLevelRemoved }?: {
+        removeCrossedLevels?: boolean;
+        onCrossedLevelRemoved?: OnLevelRemovedCB;
+    });
     update(bookChange: BookChange): void;
     bestBid(): BookPriceLevel | undefined;
     bestAsk(): BookPriceLevel | undefined;
-    removeBestAsk(): void;
-    removeBestBid(): void;
+    private _removeCrossedLevelsIfNeeded;
+    private _removeBestAsk;
+    private _removeBestBid;
     bids(): IterableIterator<BookPriceLevel>;
@@ -12,0 +20,0 @@ asks(): IterableIterator<BookPriceLevel>;

dist/orderbook.js

@@ -6,6 +6,8 @@ "use strict";
 class OrderBook {
-    constructor() {
+    constructor({ removeCrossedLevels, onCrossedLevelRemoved } = {}) {
         this._bids = new bintrees_1.RBTree((nodeA, nodeB) => nodeB.price - nodeA.price);
         this._asks = new bintrees_1.RBTree((nodeA, nodeB) => nodeA.price - nodeB.price);
         this._receivedInitialSnapshot = false;
+        this._removeCrossedLevels = removeCrossedLevels;
+        this._onCrossedLevelRemoved = onCrossedLevelRemoved;
     }
@@ -24,2 +26,5 @@ update(bookChange) {
         }
+        if (this._removeCrossedLevels) {
+            this._removeCrossedLevelsIfNeeded(bookChange);
+        }
     }
@@ -40,3 +45,36 @@ bestBid() {
     }
-    removeBestAsk() {
+    _removeCrossedLevelsIfNeeded(bookChange) {
+        let bestBid = this.bestBid();
+        let bestAsk = this.bestAsk();
+        let bookIsCrossed = bestBid !== undefined && bestAsk !== undefined && bestBid.price >= bestAsk.price;
+        // if after update we have crossed order book (best bid >= best ask)
+        // it most likely means that exchange has not published delete message for the other side of the book
+        // more info:
+        // https://www.reddit.com/r/KrakenSupport/comments/d1a4nx/websocket_orderbook_receiving_wrong_bid_price_for/
+        // https://www.reddit.com/r/BitMEX/comments/8lbj9e/bidask_ledger_weirdness/
+        // https://twitter.com/coinarb/status/931260529993170944
+        if (bookIsCrossed) {
+            // decide from which side of the book we should remove level so book isn't crossed anymore
+            // if current book update updated "best ask" it means we should remove "best bid" as exchange hasn't provided book change update
+            // that deletes it, and vice versa for for "best bids"
+            const shouldRemoveBestBid = bookChange.asks.some((s) => s.price === bestAsk.price);
+            while (bookIsCrossed) {
+                if (shouldRemoveBestBid) {
+                    this._removeBestBid();
+                }
+                else {
+                    this._removeBestAsk();
+                }
+                const newBestBid = this.bestBid();
+                const newBestAsk = this.bestAsk();
+                if (this._onCrossedLevelRemoved !== undefined) {
+                    this._onCrossedLevelRemoved(bookChange, bestBid, newBestBid, bestAsk, newBestAsk);
+                }
+                bestBid = newBestBid;
+                bestAsk = newBestAsk;
+                bookIsCrossed = bestBid !== undefined && bestAsk !== undefined && bestBid.price >= bestAsk.price;
+            }
+        }
+    }
+    _removeBestAsk() {
         const bestAsk = this.bestAsk();
@@ -52,3 +90,3 @@ if (bestAsk !== undefined) {
     }
-    removeBestBid() {
+    _removeBestBid() {
         const bestBid = this.bestBid();
@@ -55,0 +93,0 @@ if (bestBid !== undefined) {

dist/realtimefeeds/huobi.d.ts

@@ -15,2 +15,3 @@ /// <reference types="node" />
     protected wssURL: string;
+    protected channelSuffixMap: any;
 }
@@ -17,0 +18,0 @@ export declare class HuobiDMRealTimeFeed extends HuobiRealTimeFeedBase {

dist/realtimefeeds/huobi.js

@@ -9,6 +9,3 @@ "use strict";
         super(...arguments);
-        this.channelSuffixMap = {
-            trade: '.detail',
-            depth: '.step0'
-        };
+        this.channelSuffixMap = {};
         this.decompress = (message) => {
@@ -55,2 +52,7 @@ message = zlib_1.unzipSync(message);
         this.wssURL = 'wss://api.huobi.pro/ws';
+        this.channelSuffixMap = {
+            trade: '.detail',
+            depth: '.step0',
+            mbp: '.150'
+        };
     }
@@ -57,0 +59,0 @@ }

dist/realtimefeeds/index.js

@@ -32,2 +32,3 @@ "use strict";
 const delta_1 = require("./delta");
+const gateio_1 = require("./gateio");
 __exportStar(require("./realtimefeed"), exports);
@@ -65,3 +66,4 @@ const realTimeFeedsMap = {
     phemex: phemex_1.PhemexRealTimeFeed,
-    delta: delta_1.DeltaRealTimeFeed
+    delta: delta_1.DeltaRealTimeFeed,
+    'gate-io': gateio_1.GateIORealTimeFeed
 };
@@ -68,0 +70,0 @@ function getRealTimeFeedFactory(exchange) {

package.json

 {
   "name": "tardis-dev",
-  "version": "10.1.15",
+  "version": "10.1.16",
   "engines": {
@@ -5,0 +5,0 @@ "node": ">=12"

src/combine.ts

@@ -31,6 +31,18 @@ import { PassThrough } from 'stream'
 
-  if (current.result.value.localTimestamp < oldest.result.value.localTimestamp) {
+  const currentTimestamp = current.result.value.localTimestamp.valueOf()
+  const oldestTimestamp = oldest.result.value.localTimestamp.valueOf()
+
+  if (currentTimestamp < oldestTimestamp) {
     return current
   }
 
+  if (currentTimestamp === oldestTimestamp) {
+    const currentTimestampMicroSeconds = current.result.value.localTimestamp.μs || 0
+    const oldestTimestampMicroSeconds = oldest.result.value.localTimestamp.μs || 0
+
+    if (currentTimestampMicroSeconds < oldestTimestampMicroSeconds) {
+      return current
+    }
+  }
+
   return oldest
@@ -37,0 +49,0 @@ }

src/computable/booksnapshot.ts

@@ -1,13 +0,5 @@
-import { OrderBook } from '../orderbook'
+import { OrderBook, OnLevelRemovedCB } from '../orderbook'
 import { BookChange, BookPriceLevel, BookSnapshot, Optional } from '../types'
 import { Computable } from './computable'
 
-type OnLevelRemovedCB = (
-  bookChange: BookChange,
-  bestBidBeforeRemoval: BookPriceLevel | undefined,
-  bestBidAfterRemoval: BookPriceLevel | undefined,
-  bestAskBeforeRemoval: BookPriceLevel | undefined,
-  bestAskAfterRemoval: BookPriceLevel | undefined
-) => void
-
 type BookSnapshotComputableOptions = {
@@ -46,8 +38,6 @@ name?: string
   private readonly _type = 'book_snapshot'
-  private readonly _orderBook = new OrderBook()
+  private readonly _orderBook: OrderBook
   private readonly _depth: number
   private readonly _interval: number
   private readonly _name: string
-  private readonly _removeCrossedLevels: boolean | undefined
-  private readonly _onCrossedLevelRemoved: OnLevelRemovedCB | undefined
 
@@ -61,5 +51,8 @@ private _lastUpdateTimestamp: Date = new Date(-1)
     this._interval = interval
-    this._removeCrossedLevels = removeCrossedLevels
-    this._onCrossedLevelRemoved = onCrossedLevelRemoved
 
+    this._orderBook = new OrderBook({
+      removeCrossedLevels,
+      onCrossedLevelRemoved
+    })
+
     // initialize all bids/asks levels to empty ones
@@ -120,39 +113,2 @@ for (let i = 0; i < this._depth; i++) {
 
-    if (this._removeCrossedLevels) {
-      let bestBid = this._orderBook.bestBid()
-      let bestAsk = this._orderBook.bestAsk()
-      let bookIsCrossed = bestBid !== undefined && bestAsk !== undefined && bestBid.price >= bestAsk.price
-      // if after update we have crossed order book (best bid >= best ask)
-      // it most likely means that exchange has not published delete message for the other side of the book
-      // more info:
-      // https://www.reddit.com/r/KrakenSupport/comments/d1a4nx/websocket_orderbook_receiving_wrong_bid_price_for/
-      // https://www.reddit.com/r/BitMEX/comments/8lbj9e/bidask_ledger_weirdness/
-      // https://twitter.com/coinarb/status/931260529993170944
-
-      if (bookIsCrossed) {
-        // decide from which side of the book we should remove level so book isn't crossed anymore
-        // if current book update updated "best ask" it means we should remove "best bid" as exchange hasn't provided book change update
-        // that deletes it, and vice versa for for "best bids"
-
-        const shouldRemoveBestBid = bookChange.asks.some((s) => s.price === bestAsk!.price)
-
-        while (bookIsCrossed) {
-          if (shouldRemoveBestBid) {
-            this._orderBook.removeBestBid()
-          } else {
-            this._orderBook.removeBestAsk()
-          }
-          const newBestBid = this._orderBook.bestBid()
-          const newBestAsk = this._orderBook.bestAsk()
-          if (this._onCrossedLevelRemoved !== undefined) {
-            this._onCrossedLevelRemoved(bookChange, bestBid, newBestBid, bestAsk, newBestAsk)
-          }
-
-          bestBid = newBestBid
-          bestAsk = newBestAsk
-          bookIsCrossed = bestBid !== undefined && bestAsk !== undefined && bestBid.price >= bestAsk.price
-        }
-      }
-    }
-
     for (let i = 0; i < this._depth; i++) {
@@ -159,0 +115,0 @@ const bidLevelResult = bidsIterable.next()

src/consts.ts

@@ -27,2 +27,3 @@ export const EXCHANGES = [
   'ftx-us',
+  'gate-io',
   'okcoin',
@@ -168,3 +169,3 @@ 'bitflyer',
 
-const HUOBI_CHANNELS = ['depth', 'detail', 'trade', 'bbo'] as const
+const HUOBI_CHANNELS = ['depth', 'detail', 'trade', 'bbo', 'mbp'] as const
 
@@ -204,2 +205,3 @@ const HUOBI_DM_CHANNELS = ['depth', 'detail', 'trade', 'bbo', 'basis', 'liquidation_orders', 'contract_info', 'open_interest'] as const
 
+const GATE_IO_CHANNELS = ['trades', 'depth', 'ticker']
 export const EXCHANGE_CHANNELS_INFO = {
@@ -237,3 +239,4 @@ bitmex: BITMEX_CHANNELS,
   phemex: PHEMEX_CHANNELS,
-  delta: DELTA_CHANNELS
+  delta: DELTA_CHANNELS,
+  'gate-io': GATE_IO_CHANNELS,
 }

src/handy.ts

@@ -226,3 +226,4 @@ import crypto, { createHash } from 'crypto'
   keepAlive: true,
-  keepAliveMsecs: 10 * ONE_SEC_IN_MS
+  keepAliveMsecs: 10 * ONE_SEC_IN_MS,
+  maxSockets: 120
 })
@@ -243,3 +244,3 @@
     agent: httpsAgent,
-    timeout: 45 * ONE_SEC_IN_MS,
+    timeout: 90 * ONE_SEC_IN_MS,
     headers: {
@@ -333,1 +334,59 @@ 'Accept-Encoding': 'gzip',
 }
+
+export class CircularBuffer<T> {
+  private _buffer: T[] = []
+  private _index: number = 0
+  constructor(private readonly _bufferSize: number) {}
+
+  append(value: T) {
+    const isFull = this._buffer.length === this._bufferSize
+    let poppedValue
+    if (isFull) {
+      poppedValue = this._buffer[this._index]
+    }
+    this._buffer[this._index] = value
+    this._index = (this._index + 1) % this._bufferSize
+
+    return poppedValue
+  }
+
+  *items() {
+    for (let i = 0; i < this._buffer.length; i++) {
+      const index = (this._index + i) % this._buffer.length
+      yield this._buffer[index]
+    }
+  }
+
+  get count() {
+    return this._buffer.length
+  }
+
+  clear() {
+    this._buffer = []
+    this._index = 0
+  }
+}
+
+export class CappedSet<T> {
+  private _set = new Set<T>()
+  constructor(private readonly _maxSize: number) {}
+
+  public has(value: T) {
+    return this._set.has(value)
+  }
+
+  public add(value: T) {
+    if (this._set.size >= this._maxSize) {
+      this._set.delete(this._set.keys().next().value)
+    }
+    this._set.add(value)
+  }
+
+  public remove(value: T) {
+    this._set.delete(value)
+  }
+
+  public size() {
+    return this._set.size
+  }
+}

src/index.ts

@@ -15,1 +15,2 @@ export * from './apikeyaccessinfo'
 export * from './types'
+export * from './filter'

src/mappers/binance.ts

 import { debug } from '../debug'
+import { CircularBuffer } from '../handy'
 import { BookChange, DerivativeTicker, Exchange, FilterForExchange, Trade } from '../types'
@@ -48,36 +49,2 @@ import { Mapper, PendingTickerInfoHelper } from './mapper'
 
-class CircularBuffer<T> {
-  private _buffer: T[] = []
-  private _index: number = 0
-  constructor(private readonly _bufferSize: number) {}
-
-  append(value: T) {
-    const isFull = this._buffer.length === this._bufferSize
-    let poppedValue
-    if (isFull) {
-      poppedValue = this._buffer[this._index]
-    }
-    this._buffer[this._index] = value
-    this._index = (this._index + 1) % this._bufferSize
-
-    return poppedValue
-  }
-
-  *items() {
-    for (let i = 0; i < this._buffer.length; i++) {
-      const index = (this._index + i) % this._buffer.length
-      yield this._buffer[index]
-    }
-  }
-
-  get count() {
-    return this._buffer.length
-  }
-
-  clear() {
-    this._buffer = []
-    this._index = 0
-  }
-}
-
 export class BinanceBookChangeMapper
@@ -84,0 +51,0 @@ implements Mapper<'binance' | 'binance-jersey' | 'binance-us' | 'binance-futures' | 'binance-delivery', BookChange> {

src/mappers/coinbase.ts

 import { BookChange, Trade } from '../types'
 import { Mapper } from './mapper'
+import { parseμs } from '../handy'
 
@@ -21,2 +22,5 @@ // https://docs.pro.coinbase.com/#websocket-feed
   *map(message: CoinbaseTrade, localTimestamp: Date): IterableIterator<Trade> {
+    const timestamp = new Date(message.time)
+    timestamp.μs = parseμs(message.time)
+
     yield {
@@ -30,3 +34,3 @@ type: 'trade',
       side: message.side === 'sell' ? 'buy' : 'sell', // coinbase side field indicates the maker order side
-      timestamp: new Date(message.time),
+      timestamp,
       localTimestamp: localTimestamp
@@ -94,2 +98,3 @@ }
       } else {
+        timestamp.μs = parseμs(message.time)
         this._symbolLastTimestampMap.set(message.product_id, timestamp)
@@ -96,0 +101,0 @@ }

src/mappers/huobi.ts

 import { BookChange, DerivativeTicker, Exchange, FilterForExchange, Trade } from '../types'
 import { Mapper, PendingTickerInfoHelper } from './mapper'
+import { CircularBuffer } from '../handy'
 
@@ -82,2 +83,5 @@ // https://huobiapi.github.io/docs/spot/v1/en/#websocket-market-data
     const asks = Array.isArray(data.asks) ? data.asks : []
+    if (bids.length === 0 && asks.length === 0) {
+      return
+    }
 
@@ -101,2 +105,131 @@ yield {
 
+function isSnapshot(message: HuobiMBPDataMessage | HuobiMBPSnapshot): message is HuobiMBPSnapshot {
+  return 'rep' in message
+}
+
+export class HuobiMBPBookChangeMapper implements Mapper<'huobi', BookChange> {
+  protected readonly symbolToMBPInfoMapping: {
+    [key: string]: MBPInfo
+  } = {}
+
+  constructor(protected readonly _exchange: Exchange) {}
+
+  canHandle(message: any) {
+    const channel = message.ch || message.rep
+    if (channel === undefined) {
+      return false
+    }
+
+    return channel.includes('.mbp.')
+  }
+
+  getFilters(symbols?: string[]) {
+    symbols = normalizeSymbols(symbols)
+
+    return [
+      {
+        channel: 'mbp',
+        symbols
+      } as const
+    ]
+  }
+
+  *map(message: HuobiMBPDataMessage | HuobiMBPSnapshot, localTimestamp: Date) {
+    const symbol = (isSnapshot(message) ? message.rep : message.ch).split('.')[1].toUpperCase()
+
+    if (this.symbolToMBPInfoMapping[symbol] === undefined) {
+      this.symbolToMBPInfoMapping[symbol] = {
+        bufferedUpdates: new CircularBuffer<HuobiMBPDataMessage>(200)
+      }
+    }
+
+    const mbpInfo = this.symbolToMBPInfoMapping[symbol]
+    const snapshotAlreadyProcessed = mbpInfo.snapshotProcessed
+
+    if (isSnapshot(message)) {
+      if (snapshotAlreadyProcessed) {
+        return
+      }
+      const snapshotBids = message.data.bids.map(this._mapBookLevel)
+      const snapshotAsks = message.data.asks.map(this._mapBookLevel)
+
+      // if there were any depth updates buffered, let's proccess those by adding to or updating the initial snapshot
+      // when prevSeqNum >= snapshot seqNum
+      for (const update of mbpInfo.bufferedUpdates.items()) {
+        if (update.tick.prevSeqNum < message.data.seqNum) {
+          continue
+        }
+
+        const bookChange = this._mapMBPUpdate(update, symbol, localTimestamp)
+        if (bookChange !== undefined) {
+          for (const bid of bookChange.bids) {
+            const matchingBid = snapshotBids.find((b) => b.price === bid.price)
+            if (matchingBid !== undefined) {
+              matchingBid.amount = bid.amount
+            } else {
+              snapshotBids.push(bid)
+            }
+          }
+
+          for (const ask of bookChange.asks) {
+            const matchingAsk = snapshotAsks.find((a) => a.price === ask.price)
+            if (matchingAsk !== undefined) {
+              matchingAsk.amount = ask.amount
+            } else {
+              snapshotAsks.push(ask)
+            }
+          }
+        }
+      }
+
+      mbpInfo.snapshotProcessed = true
+      mbpInfo.bufferedUpdates.clear()
+
+      yield {
+        type: 'book_change',
+        symbol,
+        exchange: this._exchange,
+        isSnapshot: true,
+        bids: snapshotBids,
+        asks: snapshotAsks,
+        timestamp: new Date(message.ts),
+        localTimestamp
+      } as const
+    } else if (snapshotAlreadyProcessed) {
+      // snapshot was already processed let's map the mbp message as normal book_change
+      const update = this._mapMBPUpdate(message, symbol, localTimestamp)
+      if (update !== undefined) {
+        yield update
+      }
+    } else {
+      // there was no snapshot yet, let's buffer the update
+      mbpInfo.bufferedUpdates.append(message)
+    }
+  }
+
+  private _mapMBPUpdate(message: HuobiMBPDataMessage, symbol: string, localTimestamp: Date) {
+    const bids = Array.isArray(message.tick.bids) ? message.tick.bids : []
+    const asks = Array.isArray(message.tick.asks) ? message.tick.asks : []
+
+    if (bids.length === 0 && asks.length === 0) {
+      return
+    }
+
+    return {
+      type: 'book_change',
+      symbol,
+      exchange: this._exchange,
+      isSnapshot: false,
+      bids: bids.map(this._mapBookLevel),
+      asks: asks.map(this._mapBookLevel),
+      timestamp: new Date(message.ts),
+      localTimestamp: localTimestamp
+    } as const
+  }
+
+  private _mapBookLevel(level: HuobiBookLevel) {
+    return { price: level[0], amount: level[1] }
+  }
+}
+
 function normalizeSymbols(symbols?: string[]) {
@@ -226,4 +359,4 @@ if (symbols !== undefined) {
         tick: {
-          bids: HuobiBookLevel[] | null
-          asks: HuobiBookLevel[] | null
+          bids?: HuobiBookLevel[] | null
+          asks?: HuobiBookLevel[] | null
           event: 'snapshot' | 'update'
@@ -260,1 +393,26 @@ }
 }
+
+type HuobiMBPDataMessage = HuobiDataMessage & {
+  ts: number
+  tick: {
+    bids?: HuobiBookLevel[] | null
+    asks?: HuobiBookLevel[] | null
+    seqNum: number
+    prevSeqNum: number
+  }
+}
+
+type HuobiMBPSnapshot = {
+  ts: number
+  rep: string
+  data: {
+    bids: HuobiBookLevel[]
+    asks: HuobiBookLevel[]
+    seqNum: number
+  }
+}
+
+type MBPInfo = {
+  bufferedUpdates: CircularBuffer<HuobiMBPDataMessage>
+  snapshotProcessed?: boolean
+}

src/mappers/index.ts

@@ -22,3 +22,3 @@ import { ONE_SEC_IN_MS } from '../handy'
 import { hitBtcBookChangeMapper, hitBtcTradesMapper } from './hitbtc'
-import { HuobiBookChangeMapper, HuobiTradesMapper, HuobiDerivativeTickerMapper } from './huobi'
+import { HuobiBookChangeMapper, HuobiTradesMapper, HuobiDerivativeTickerMapper, HuobiMBPBookChangeMapper } from './huobi'
 import { krakenBookChangeMapper, krakenTradesMapper } from './kraken'
@@ -28,2 +28,3 @@ import { Mapper } from './mapper'
 import { phemexBookChangeMapper, PhemexDerivativeTickerMapper, phemexTradesMapper } from './phemex'
+import { GateBookChangeMapper, GateIOTradesMapper } from './gateio'
 
@@ -72,3 +73,4 @@ export * from './mapper'
   phemex: () => phemexTradesMapper,
-  delta: () => deltaTradesMapper
+  delta: () => deltaTradesMapper,
+  'gate-io': () => new GateIOTradesMapper('gate-io')
 }
@@ -105,3 +107,7 @@
     new OkexBookChangeMapper('okex-options', 'option', localTimestamp.valueOf() >= new Date('2020-02-08').valueOf()),
-  huobi: () => new HuobiBookChangeMapper('huobi'),
+  huobi: (localTimestamp: Date) =>
+    localTimestamp.valueOf() >= new Date('2020-07-03').valueOf()
+      ? new HuobiMBPBookChangeMapper('huobi')
+      : new HuobiBookChangeMapper('huobi'),
+
   'huobi-dm': () => new HuobiBookChangeMapper('huobi-dm'),
@@ -114,3 +120,4 @@ 'huobi-dm-swap': () => new HuobiBookChangeMapper('huobi-dm-swap'),
   phemex: () => phemexBookChangeMapper,
-  delta: () => deltaBookChangeMapper
+  delta: () => deltaBookChangeMapper,
+  'gate-io': () => new GateBookChangeMapper('gate-io')
 }
@@ -117,0 +124,0 @@

src/mappers/okex.ts

@@ -96,2 +96,6 @@ import { BookChange, DerivativeTicker, Exchange, FilterForExchange, Trade, OptionSummary } from '../types'
     for (const message of okexDepthDataMessage.data) {
+      if (message.bids.length === 0 && message.asks.length === 0) {
+        continue
+      }
+
       yield {
@@ -98,0 +102,0 @@ type: 'book_change',

src/orderbook.ts

 import { RBTree } from 'bintrees'
 import { BookChange, BookPriceLevel, Writeable } from './types'
 
+export type OnLevelRemovedCB = (
+  bookChange: BookChange,
+  bestBidBeforeRemoval: BookPriceLevel | undefined,
+  bestBidAfterRemoval: BookPriceLevel | undefined,
+  bestAskBeforeRemoval: BookPriceLevel | undefined,
+  bestAskAfterRemoval: BookPriceLevel | undefined
+) => void
+
 export class OrderBook {
   private readonly _bids = new RBTree<BookPriceLevel>((nodeA, nodeB) => nodeB.price - nodeA.price)
   private readonly _asks = new RBTree<BookPriceLevel>((nodeA, nodeB) => nodeA.price - nodeB.price)
+  private readonly _removeCrossedLevels: boolean | undefined
+  private readonly _onCrossedLevelRemoved: OnLevelRemovedCB | undefined
 
   private _receivedInitialSnapshot = false
 
+  constructor({
+    removeCrossedLevels,
+    onCrossedLevelRemoved
+  }: { removeCrossedLevels?: boolean; onCrossedLevelRemoved?: OnLevelRemovedCB } = {}) {
+    this._removeCrossedLevels = removeCrossedLevels
+    this._onCrossedLevelRemoved = onCrossedLevelRemoved
+  }
+
   public update(bookChange: BookChange) {
@@ -22,2 +40,6 @@ // clear everything up, when snapshot received so we don't have stale levels by accident
     }
+
+    if (this._removeCrossedLevels) {
+      this._removeCrossedLevelsIfNeeded(bookChange)
+    }
   }
@@ -43,3 +65,41 @@
 
-  public removeBestAsk() {
+  private _removeCrossedLevelsIfNeeded(bookChange: BookChange) {
+    let bestBid = this.bestBid()
+    let bestAsk = this.bestAsk()
+    let bookIsCrossed = bestBid !== undefined && bestAsk !== undefined && bestBid.price >= bestAsk.price
+    // if after update we have crossed order book (best bid >= best ask)
+    // it most likely means that exchange has not published delete message for the other side of the book
+    // more info:
+    // https://www.reddit.com/r/KrakenSupport/comments/d1a4nx/websocket_orderbook_receiving_wrong_bid_price_for/
+    // https://www.reddit.com/r/BitMEX/comments/8lbj9e/bidask_ledger_weirdness/
+    // https://twitter.com/coinarb/status/931260529993170944
+
+    if (bookIsCrossed) {
+      // decide from which side of the book we should remove level so book isn't crossed anymore
+      // if current book update updated "best ask" it means we should remove "best bid" as exchange hasn't provided book change update
+      // that deletes it, and vice versa for for "best bids"
+
+      const shouldRemoveBestBid = bookChange.asks.some((s) => s.price === bestAsk!.price)
+
+      while (bookIsCrossed) {
+        if (shouldRemoveBestBid) {
+          this._removeBestBid()
+        } else {
+          this._removeBestAsk()
+        }
+
+        const newBestBid = this.bestBid()
+        const newBestAsk = this.bestAsk()
+        if (this._onCrossedLevelRemoved !== undefined) {
+          this._onCrossedLevelRemoved(bookChange, bestBid, newBestBid, bestAsk, newBestAsk)
+        }
+
+        bestBid = newBestBid
+        bestAsk = newBestAsk
+        bookIsCrossed = bestBid !== undefined && bestAsk !== undefined && bestBid.price >= bestAsk.price
+      }
+    }
+  }
+
+  private _removeBestAsk() {
     const bestAsk = this.bestAsk()
@@ -57,3 +117,3 @@
 
-  public removeBestBid() {
+  private _removeBestBid() {
     const bestBid = this.bestBid()
@@ -60,0 +120,0 @@

src/realtimefeeds/huobi.ts

@@ -7,6 +7,3 @@ import { unzipSync } from 'zlib'
   protected abstract wssURL: string
-  protected channelSuffixMap = {
-    trade: '.detail',
-    depth: '.step0'
-  } as any
+  protected channelSuffixMap = {} as any
 
@@ -60,2 +57,8 @@ protected mapToSubscribeMessages(filters: Filter<string>[]): any[] {
   protected wssURL = 'wss://api.huobi.pro/ws'
+
+  protected channelSuffixMap = {
+    trade: '.detail',
+    depth: '.step0',
+    mbp: '.150'
+  } as any
 }
@@ -62,0 +65,0 @@

src/realtimefeeds/index.ts

@@ -27,2 +27,3 @@ import { Exchange, Filter } from '../types'
 import { DeltaRealTimeFeed } from './delta'
+import { GateIORealTimeFeed } from './gateio'
 
@@ -64,3 +65,4 @@ export * from './realtimefeed'
   phemex: PhemexRealTimeFeed,
-  delta: DeltaRealTimeFeed
+  delta: DeltaRealTimeFeed,
+  'gate-io': GateIORealTimeFeed
 }
@@ -67,0 +69,0 @@

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

842039

increased by7.64%

Number of package files

324

increased by6.58%

Lines of code

15139

increased by8.66%

No dependency changes