tardis-machine - npm Package Compare versions

Comparing version 1.3.5 to 2.0.0-alpha.0

dist/mappers.d.ts

@@ -1,9 +0,6 @@
-import { Filter, Exchange } from 'tardis-client';
-export declare const subscriptionsMappers: {
-    [key in Exchange]?: SubscriptionMapper;
-};
+export declare const subscriptionsMappers: any;
 export declare type SubscriptionMapper = {
     canHandle: (message: object) => boolean;
-    map: (message: object) => Filter<any>[];
+    map: (message: object) => any[];
 };
 //# sourceMappingURL=mappers.d.ts.map

dist/replaysession.js

@@ -103,3 +103,4 @@ "use strict";
             to: queryString['to'],
-            filters: []
+            filters: [],
+            skipDecoding: true
         };
@@ -156,3 +157,3 @@ if (!mappers_1.subscriptionsMappers[exchange]) {
         // optimized sendAll method that tries to send  all data feed messages as fast as possible without synchronization etc
-        const dataFeedMessages = this._tardisClient.replayRaw(this._replayOptions);
+        const dataFeedMessages = this._tardisClient.replay(this._replayOptions);
         let buffered = [];
@@ -199,3 +200,3 @@ for await (let { message } of dataFeedMessages) {
     async *_sendDataFeedIterable() {
-        const dataFeedMessages = this._tardisClient.replayRaw(this._replayOptions);
+        const dataFeedMessages = this._tardisClient.replay(this._replayOptions);
         for await (let { message, localTimestamp } of dataFeedMessages) {
@@ -202,0 +203,0 @@ this._checkConnection();

dist/tardismachine.js

@@ -59,2 +59,53 @@ "use strict";
         });
+        router.on('GET', '/replay-normalized', async (req, res) => {
+            const parsedQuery = url_1.default.parse(req.url, true).query;
+            const dataTypeFromQuery = parsedQuery['datatype'];
+            const provideSnapshots = dataTypeFromQuery === 'book_snapshot.1m';
+            const normalizedMessages = this._tardisClient.replayNormalized({
+                exchange: parsedQuery['exchange'],
+                dataTypes: provideSnapshots ? 'book_change' : dataTypeFromQuery,
+                from: parsedQuery['from'],
+                to: parsedQuery['to'],
+                symbols: [parsedQuery['symbol']]
+            });
+            res.setHeader('Content-Type', 'application/json');
+            const orderBook = new tardis_client_1.OrderBook();
+            let lastMessageTimestamp;
+            for await (const message of normalizedMessages) {
+                if (provideSnapshots) {
+                    if (!lastMessageTimestamp) {
+                        lastMessageTimestamp = message.localTimestamp;
+                    }
+                    if (message.localTimestamp.getUTCMinutes() !== lastMessageTimestamp.getUTCMinutes() ||
+                        message.localTimestamp.getUTCHours() !== lastMessageTimestamp.getUTCHours()) {
+                        const timestamp = new Date(lastMessageTimestamp);
+                        timestamp.setUTCSeconds(0, 0);
+                        const snapshot = {
+                            type: 'book_snapshot.1m',
+                            symbol: message.symbol,
+                            timestamp,
+                            timestampReal: lastMessageTimestamp,
+                            asks: Array.from(orderBook.asks()),
+                            bids: Array.from(orderBook.bids())
+                        };
+                        const ok = res.write(`${JSON.stringify(snapshot)}\n`);
+                        // let's handle backpressure - https://nodejs.org/api/http.html#http_request_write_chunk_encoding_callback
+                        if (!ok) {
+                            await events_1.once(res, 'drain');
+                        }
+                    }
+                    orderBook.update(message);
+                    lastMessageTimestamp = message.localTimestamp;
+                }
+                else {
+                    const ok = res.write(`${JSON.stringify(message)}\n`);
+                    // let's handle backpressure - https://nodejs.org/api/http.html#http_request_write_chunk_encoding_callback
+                    if (!ok) {
+                        await events_1.once(res, 'drain');
+                    }
+                }
+            }
+            console.log('end');
+            res.end('');
+        });
         // set timeout to 0 meaning infinite http timout - streaming may take some time expecially for longer date ranges
@@ -95,7 +146,5 @@ this._httpServer.timeout = 0;
         else {
-            console.log(`TardisMachine is running...`);
-            console.log(`--> HTTP endpoint: http://localhost:${port}/replay`);
-            console.log(`--> WebSocket endpoint: http://localhost:${port}/ws-replay`);
+            console.log(`TardisMachine is running on ${port} port...`);
         }
-        console.log(`Check out https://tardis.dev for help or more information.`);
+        console.log(`Check out https://docs.tardis.dev for help or more information.`);
     }
@@ -136,3 +185,3 @@ async stop() {
     }
-    async _writeDataFeedMessagesToResponse(res, options) {
+    async _writeDataFeedMessagesToResponse(res, { exchange, from, to, filters }) {
         const responsePrefixBuffer = Buffer.from('{"localTimestamp":"');
@@ -146,3 +195,3 @@ const responseMiddleBuffer = Buffer.from('","message":');
         let totalMessagesCount = 0;
-        const dataFeedMessages = this._tardisClient.replayRaw(options);
+        const dataFeedMessages = this._tardisClient.replay({ exchange, from, to, filters, skipDecoding: true });
         for await (let { message, localTimestamp } of dataFeedMessages) {
@@ -149,0 +198,0 @@ totalMessagesCount++;

package.json

 {
   "name": "tardis-machine",
-  "version": "1.3.5",
+  "version": "2.0.0-alpha.0",
   "engines": {
@@ -48,3 +48,3 @@ "node": ">=12"
     "is-docker": "^2.0.0",
-    "tardis-client": "^1.2.4",
+    "tardis-client": "^2.0.0-alpha.0",
     "yargs": "^13.2.4",
@@ -51,0 +51,0 @@ "ws": "^7.0.0"

src/mappers.ts

@@ -1,3 +0,1 @@
-import { Filter, Exchange } from 'tardis-client'
-
 // https://www.bitmex.com/app/wsAPI
@@ -35,3 +33,3 @@ const bitmexMapper: SubscriptionMapper = {
     const topLevelSymbols = message.product_ids
-    const finalChannels: Filter<any>[] = []
+    const finalChannels: any[] = []
 
@@ -192,3 +190,3 @@ const channelMappings = {
   map: (message: any) => {
-    const finalChannels: Filter<any>[] = []
+    const finalChannels: any[] = []
 
@@ -214,3 +212,3 @@ const channelMappings = {
 
-export const subscriptionsMappers: { [key in Exchange]?: SubscriptionMapper } = {
+export const subscriptionsMappers: any = {
   bitmex: bitmexMapper,
@@ -230,3 +228,3 @@ coinbase: coinbaseMaper,
   canHandle: (message: object) => boolean
-  map: (message: object) => Filter<any>[]
+  map: (message: object) => any[]
 }

src/replaysession.ts

 import { ParsedUrlQuery } from 'querystring'
 import dbg from 'debug'
 import WebSocket from 'ws'
-import { TardisClient, Exchange, ReplayOptions } from 'tardis-client'
+import { TardisClient, ReplayOptions } from 'tardis-client'
 import { subscriptionsMappers, SubscriptionMapper } from './mappers'
@@ -108,3 +108,3 @@
 export class WebsocketConnection {
-  private readonly _replayOptions: ReplayOptions<any>
+  private readonly _replayOptions: ReplayOptions<any, true>
   private readonly _subscriptionsMapper: SubscriptionMapper
@@ -117,3 +117,3 @@ private _subscriptionsCount = 0
   constructor(private readonly _websocket: WebSocket, private readonly _tardisClient: TardisClient, queryString: ParsedUrlQuery) {
-    const exchange = queryString['exchange'] as Exchange
+    const exchange = queryString['exchange'] as any
     this._replayOptions = {
@@ -123,3 +123,4 @@ exchange,
       to: queryString['to'] as string,
-      filters: []
+      filters: [],
+      skipDecoding: true
     }
@@ -190,3 +191,3 @@ if (!subscriptionsMappers[exchange]) {
     // optimized sendAll method that tries to send  all data feed messages as fast as possible without synchronization etc
-    const dataFeedMessages = this._tardisClient.replayRaw(this._replayOptions)
+    const dataFeedMessages = this._tardisClient.replay(this._replayOptions)
     let buffered: Buffer[] = []
@@ -241,3 +242,3 @@
   private async *_sendDataFeedIterable() {
-    const dataFeedMessages = this._tardisClient.replayRaw(this._replayOptions)
+    const dataFeedMessages = this._tardisClient.replay(this._replayOptions)
 
@@ -244,0 +245,0 @@ for await (let { message, localTimestamp } of dataFeedMessages) {

src/tardismachine.ts

@@ -8,3 +8,3 @@ import http, { OutgoingMessage, IncomingMessage } from 'http'
 import WebSocket from 'ws'
-import { TardisClient, ReplayOptions, Exchange } from 'tardis-client'
+import { TardisClient, OrderBook } from 'tardis-client'
 import { ReplaySession, WebsocketConnection } from './replaysession'
@@ -37,3 +37,3 @@
 
-        const replayOptions: ReplayOptions<any> = {
+        const replayOptions = {
           exchange: parsedQuery['exchange'] as string,
@@ -73,2 +73,64 @@ from: parsedQuery['from'] as string,
 
+    router.on('GET', '/replay-normalized', async (req, res) => {
+      const parsedQuery = url.parse(req.url!, true).query
+      const dataTypeFromQuery = parsedQuery['datatype'] as any
+      const provideSnapshots = dataTypeFromQuery === 'book_snapshot.1m'
+
+      const normalizedMessages = this._tardisClient.replayNormalized({
+        exchange: parsedQuery['exchange'] as any,
+        dataTypes: provideSnapshots ? 'book_change' : dataTypeFromQuery,
+        from: parsedQuery['from'] as string,
+        to: parsedQuery['to'] as string,
+        symbols: [parsedQuery['symbol'] as any]
+      })
+
+      res.setHeader('Content-Type', 'application/json')
+
+      const orderBook = new OrderBook()
+      let lastMessageTimestamp: Date | undefined
+
+      for await (const message of normalizedMessages) {
+        if (provideSnapshots) {
+          if (!lastMessageTimestamp) {
+            lastMessageTimestamp = message.localTimestamp
+          }
+
+          if (
+            (message.localTimestamp as Date).getUTCMinutes() !== lastMessageTimestamp!.getUTCMinutes() ||
+            (message.localTimestamp as Date).getUTCHours() !== lastMessageTimestamp!.getUTCHours()
+          ) {
+            const timestamp = new Date(lastMessageTimestamp!)
+            timestamp.setUTCSeconds(0, 0)
+            const snapshot = {
+              type: 'book_snapshot.1m',
+              symbol: message.symbol,
+              timestamp,
+              timestampReal: lastMessageTimestamp,
+              asks: Array.from(orderBook.asks()),
+              bids: Array.from(orderBook.bids())
+            }
+
+            const ok = res.write(`${JSON.stringify(snapshot)}\n`)
+
+            // let's handle backpressure - https://nodejs.org/api/http.html#http_request_write_chunk_encoding_callback
+            if (!ok) {
+              await once(res, 'drain')
+            }
+          }
+
+          orderBook.update(message)
+          lastMessageTimestamp = message.localTimestamp
+        } else {
+          const ok = res.write(`${JSON.stringify(message)}\n`)
+
+          // let's handle backpressure - https://nodejs.org/api/http.html#http_request_write_chunk_encoding_callback
+          if (!ok) {
+            await once(res, 'drain')
+          }
+        }
+      }
+      console.log('end')
+      res.end('')
+    })
+
     // set timeout to 0 meaning infinite http timout - streaming may take some time expecially for longer date ranges
@@ -112,8 +174,6 @@ this._httpServer.timeout = 0
     } else {
-      console.log(`TardisMachine is running...`)
-      console.log(`--> HTTP endpoint: http://localhost:${port}/replay`)
-      console.log(`--> WebSocket endpoint: http://localhost:${port}/ws-replay`)
+      console.log(`TardisMachine is running on ${port} port...`)
     }
 
-    console.log(`Check out https://tardis.dev for help or more information.`)
+    console.log(`Check out https://docs.tardis.dev for help or more information.`)
   }
@@ -162,3 +222,6 @@
 
-  private async _writeDataFeedMessagesToResponse<T extends Exchange>(res: OutgoingMessage, options: ReplayOptions<T>) {
+  private async _writeDataFeedMessagesToResponse(
+    res: OutgoingMessage,
+    { exchange, from, to, filters }: { exchange: any; from: string; to: string; filters: any }
+  ) {
     const responsePrefixBuffer = Buffer.from('{"localTimestamp":"')
@@ -175,3 +238,3 @@ const responseMiddleBuffer = Buffer.from('","message":')
 
-    const dataFeedMessages = this._tardisClient.replayRaw(options)
+    const dataFeedMessages = this._tardisClient.replay({ exchange, from, to, filters, skipDecoding: true })
     for await (let { message, localTimestamp } of dataFeedMessages) {
@@ -178,0 +241,0 @@ totalMessagesCount++

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

No v1

Quality

Package is not semver >=1. This means it is not stable and does not support ^ ranges.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

113740

increased by6.11%

Lines of code

1485

increased by7.22%

Worsened metrics

Number of low quality alerts

1

increased byInfinity%

Dependency changes

• + Added@sindresorhus/is@2.1.1(transitive)

• + Added@szmarczak/http-timer@4.0.6(transitive)

• + Added@types/cacheable-request@6.0.3(transitive)

• + Added@types/http-cache-semantics@4.0.4(transitive)

• + Added@types/keyv@3.1.4(transitive)

• + Added@types/responselike@1.0.3(transitive)

• + Addedaggregate-error@3.1.0(transitive)

• + Addedcacheable-lookup@2.0.1(transitive)

• + Addedcacheable-request@7.0.4(transitive)

• + Addedclean-stack@2.2.0(transitive)

• + Addedclone-response@1.0.3(transitive)

• + Addedcollections@2.0.3(transitive)

• + Addeddecompress-response@5.0.0(transitive)

• + Addeddefer-to-connect@2.0.1(transitive)

• + Addedduplexer3@0.1.5(transitive)

• + Addedend-of-stream@1.4.4(transitive)

• + Addedget-stream@5.2.0(transitive)

• + Addedgot@10.7.0(transitive)

• + Addedhttp-cache-semantics@4.1.1(transitive)

• + Addedindent-string@4.0.0(transitive)

• + Addedjson-buffer@3.0.1(transitive)

• + Addedkeyv@4.5.4(transitive)

• + Addedlowercase-keys@2.0.0(transitive)

• + Addedmimic-response@1.0.12.1.0(transitive)

• + Addedmini-map@1.0.0(transitive)

• + Addednormalize-url@6.1.0(transitive)

• + Addedonce@1.4.0(transitive)

• + Addedp-cancelable@2.1.1(transitive)

• + Addedp-event@4.2.0(transitive)

• + Addedp-finally@1.0.0(transitive)

• + Addedp-map@3.0.0(transitive)

• + Addedp-timeout@3.2.0(transitive)

• + Addedpop-arrayify@1.0.0(transitive)

• + Addedpop-clear@1.0.0(transitive)

• + Addedpop-clone@1.0.1(transitive)

• + Addedpop-compare@1.0.0(transitive)

• + Addedpop-equals@1.0.0(transitive)

• + Addedpop-has@1.0.0(transitive)

• + Addedpop-hash@1.0.1(transitive)

• + Addedpop-iterate@1.0.1(transitive)

• + Addedpop-observe@2.0.2(transitive)

• + Addedpop-swap@1.0.0(transitive)

• + Addedpop-zip@1.0.0(transitive)

• + Addedpump@3.0.2(transitive)

• + Addedregexp-escape@0.0.1(transitive)

• + Addedresponselike@2.0.1(transitive)

• + Addedtardis-client@2.0.0-alpha.0(transitive)

• + Addedto-readable-stream@2.1.0(transitive)

• + Addedtype-fest@0.10.0(transitive)

• + Addedweak-map@1.0.8(transitive)

• + Addedwrappy@1.0.2(transitive)

• - Removedp-map@2.1.0(transitive)

• - Removedtardis-client@1.2.4(transitive)

• Updatedtardis-client@^2.0.0-alpha.0