Huge News!Announcing our $40M Series B led by Abstract Ventures.Learn More →
Socket
Sign inDemoInstall
Socket
Sign inDemoInstall

serum-vial

Package Overview
Dependencies
Maintainers
1
Versions
99
Alerts
File Explorer

Advanced tools

License
Socket logo

Install Socket

Detect and block malicious and high-risk dependencies

Install

serum-vial - npm Package Compare versions

Comparing version 0.7.1 to 0.8.0

11

bin/serum-vial.js

@@ -24,3 +24,3 @@ #!/usr/bin/env node

type: 'string',
describe: 'Solana node endpoint',
describe: 'Solana RPC node endpoint that serum-vial uses as a data source',
default: DEFAULT_NODE_ENDPOINT

@@ -31,3 +31,3 @@ })

type: 'string',
describe: 'Enable debug logs.',
describe: 'Log level',
choices: ['debug', 'info', 'warn', 'error'],

@@ -38,3 +38,4 @@ default: 'info'

type: 'number',
describe: 'Minions worker threads count (handle WS pub/sub)',
describe:
'Minions worker threads count that are responsible for broadcasting normalized WS messages to connected clients',
default: 1

@@ -45,3 +46,3 @@ })

type: 'boolean',
describe: 'turn on validation of L3 diffs correctness (can impact perf)',
describe: 'Turns on validation of L3 diffs correctness (can impact perf)',
default: false

@@ -59,3 +60,3 @@ })

type: 'string',
describe: 'Custom market.json definition file',
describe: 'Path to custom market.json definition file',
default: ''

@@ -62,0 +63,0 @@ })

10

dist/boot_server.js

@@ -16,5 +16,5 @@ "use strict";

const MINIONS_COUNT = os_1.default.platform() === 'linux' ? minionsCount : 1;
let readyMonionsCount = 0;
let readyMinionsCount = 0;
logger_1.logger.log('info', MINIONS_COUNT === 1 ? 'Starting single minion worker...' : `Starting ${MINIONS_COUNT} minion workers...`);
helpers_1.minionReadyChannel.onmessage = () => readyMonionsCount++;
helpers_1.minionReadyChannel.onmessage = () => readyMinionsCount++;
// start minions workers and wait until all are ready

@@ -26,3 +26,3 @@ for (let i = 0; i < MINIONS_COUNT; i++) {

minionWorker.on('error', (err) => {
logger_1.logger.log('error', `Minion worker ${minionWorker.threadId} error occured: ${err.message} ${err.stack}`);
logger_1.logger.log('error', `Minion worker ${minionWorker.threadId} error occurred: ${err.message} ${err.stack}`);
throw err;

@@ -36,3 +36,3 @@ });

while (true) {
if (readyMonionsCount === MINIONS_COUNT) {
if (readyMinionsCount === MINIONS_COUNT) {
break;

@@ -52,3 +52,3 @@ }

serumProducerWorker.on('error', (err) => {
logger_1.logger.log('error', `Serum producer worker ${serumProducerWorker.threadId} error occured: ${err.message} ${err.stack}`);
logger_1.logger.log('error', `Serum producer worker ${serumProducerWorker.threadId} error occurred: ${err.message} ${err.stack}`);
throw err;

@@ -55,0 +55,0 @@ });

2

dist/data_mapper.d.ts

@@ -9,3 +9,3 @@ import { Market } from '@project-serum/serum';

private _bidsAccountSlabItems;
private _asksAccounSlabItems;
private _asksAccountSlabItems;
private _localBidsOrders;

@@ -12,0 +12,0 @@ private _localAsksOrders;

34

dist/data_mapper.js

@@ -14,3 +14,3 @@ "use strict";

this._bidsAccountSlabItems = undefined;
this._asksAccounSlabItems = undefined;
this._asksAccountSlabItems = undefined;
// _local* are used only for verification purposes

@@ -85,3 +85,3 @@ this._localBidsOrders = undefined;

}
this._asksAccounSlabItems = newAsksSlabItems;
this._asksAccountSlabItems = newAsksSlabItems;
this._asksAccountOrders = newAsksOrders;

@@ -115,3 +115,3 @@ }

type: 'l3snapshot',
symbol: this._options.symbol,
market: this._options.symbol,
timestamp,

@@ -132,3 +132,3 @@ slot,

this._currentL2Snapshot = {
asks: this._mapToL2Snapshot(this._asksAccounSlabItems),
asks: this._mapToL2Snapshot(this._asksAccountSlabItems),
bids: this._mapToL2Snapshot(this._bidsAccountSlabItems)

@@ -138,3 +138,3 @@ };

type: 'l2snapshot',
symbol: this._options.symbol,
market: this._options.symbol,
timestamp,

@@ -152,3 +152,3 @@ slot,

type: 'quote',
symbol: this._options.symbol,
market: this._options.symbol,
timestamp,

@@ -167,3 +167,3 @@ slot,

asks: accountsData.asks !== undefined
? this._mapToL2Snapshot(this._asksAccounSlabItems)
? this._mapToL2Snapshot(this._asksAccountSlabItems)
: this._currentL2Snapshot.asks,

@@ -188,3 +188,3 @@ bids: accountsData.bids !== undefined

type: 'trade',
symbol: this._options.symbol,
market: this._options.symbol,
timestamp,

@@ -202,3 +202,3 @@ slot,

type: 'recent_trades',
symbol: this._options.symbol,
market: this._options.symbol,
timestamp,

@@ -217,3 +217,3 @@ trades: [...this._recentTrades.items()]

type: 'l2snapshot',
symbol: this._options.symbol,
market: this._options.symbol,
timestamp,

@@ -227,3 +227,3 @@ slot,

type: 'l2update',
symbol: this._options.symbol,
market: this._options.symbol,
timestamp,

@@ -245,3 +245,3 @@ slot,

type: 'quote',
symbol: this._options.symbol,
market: this._options.symbol,
timestamp,

@@ -397,3 +397,3 @@ slot,

}
// remove from currrent levels map so we know that such level exists in new levels
// remove from current levels map so we know that such level exists in new levels
currentLevelsMap.delete(newLevel[0]);

@@ -435,3 +435,3 @@ }

type: message.type,
symbol: message.symbol,
market: message.market,
publish,

@@ -456,3 +456,3 @@ payload: JSON.stringify(message),

type: 'fill',
symbol: this._options.symbol,
market: this._options.symbol,
timestamp,

@@ -480,3 +480,3 @@ slot,

type: 'done',
symbol: this._options.symbol,
market: this._options.symbol,
timestamp,

@@ -534,3 +534,3 @@ slot,

type,
symbol: this._options.symbol,
market: this._options.symbol,
timestamp,

@@ -537,0 +537,0 @@ slot,

2

dist/index.d.ts
export { bootServer, stopServer } from './boot_server';
export { getDefaultMarkets } from './helpers';
export { logger } from './logger';
export { getDefaultMarkets } from './helpers';
export * from './types';
//# sourceMappingURL=index.d.ts.map

6

dist/index.js

@@ -13,11 +13,11 @@ "use strict";

Object.defineProperty(exports, "__esModule", { value: true });
exports.getDefaultMarkets = exports.logger = exports.stopServer = exports.bootServer = void 0;
exports.logger = exports.getDefaultMarkets = exports.stopServer = exports.bootServer = void 0;
var boot_server_1 = require("./boot_server");
Object.defineProperty(exports, "bootServer", { enumerable: true, get: function () { return boot_server_1.bootServer; } });
Object.defineProperty(exports, "stopServer", { enumerable: true, get: function () { return boot_server_1.stopServer; } });
var helpers_1 = require("./helpers");
Object.defineProperty(exports, "getDefaultMarkets", { enumerable: true, get: function () { return helpers_1.getDefaultMarkets; } });
var logger_1 = require("./logger");
Object.defineProperty(exports, "logger", { enumerable: true, get: function () { return logger_1.logger; } });
var helpers_1 = require("./helpers");
Object.defineProperty(exports, "getDefaultMarkets", { enumerable: true, get: function () { return helpers_1.getDefaultMarkets; } });
__exportStar(require("./types"), exports);
//# sourceMappingURL=index.js.map

14

dist/minion.js

@@ -67,3 +67,3 @@ "use strict";

const serumMarket = {
symbol: market.name,
name: market.name,
baseCurrency: baseCurrency,

@@ -141,3 +141,3 @@ quoteCurrency: quoteCurrency,

async processMessage(message) {
const topic = `${message.type}/${message.symbol}`;
const topic = `${message.type}/${message.market}`;
if (logger_1.logger.level === 'debug') {

@@ -148,12 +148,12 @@ const diff = new Date().valueOf() - new Date(message.timestamp).valueOf();

if (message.type === 'l2snapshot') {
this._l2SnapshotsSerialized[message.symbol] = message.payload;
this._l2SnapshotsSerialized[message.market] = message.payload;
}
if (message.type === 'l3snapshot') {
this._l3SnapshotsSerialized[message.symbol] = message.payload;
this._l3SnapshotsSerialized[message.market] = message.payload;
}
if (message.type === 'quote') {
this._quotesSerialized[message.symbol] = message.payload;
this._quotesSerialized[message.market] = message.payload;
}
if (message.type === 'recent_trades') {
this._recentTradesSerialized[message.symbol] = message.payload;
this._recentTradesSerialized[message.market] = message.payload;
}

@@ -215,3 +215,3 @@ if (message.publish) {

type: 'recent_trades',
symbol: market,
market,
timestamp: new Date().toISOString(),

@@ -218,0 +218,0 @@ trades: []

10

dist/rpc_client.js

@@ -8,4 +8,4 @@ "use strict";

const web3_js_1 = require("@solana/web3.js");
const abort_controller_1 = __importDefault(require("abort-controller"));
const node_fetch_1 = __importDefault(require("node-fetch"));
const abort_controller_1 = __importDefault(require("abort-controller"));
const stream_1 = require("stream");

@@ -104,3 +104,3 @@ const ws_1 = __importDefault(require("ws"));

exports.RPCClient = RPCClient;
// this helper class handles RPC subscriptions to seprate DEX accounts (bids, asks & event queue)
// this helper class handles RPC subscriptions to separate DEX accounts (bids, asks & event queue)
// and provide notification in synchronized fashion, meaning we get at most one notification per slot

@@ -216,3 +216,3 @@ // with accounts data that changed in that slot

this._monitorConnectionIfStale(ws);
logger_1.logger.log('info', 'Estabilished new RPC WebSocket connection...', { market: this._options.marketName });
logger_1.logger.log('info', 'Established new RPC WebSocket connection...', { market: this._options.marketName });
};

@@ -252,3 +252,3 @@ ws.onmessage = (event) => {

if (message.method === 'slotNotification') {
// ignore slot notficiations which are only used as a heartbeat message
// ignore slot notifications which are only used as a heartbeat message
return;

@@ -461,3 +461,3 @@ }

// in case we fetched accounts data via REST API and WS account notification is published for such snapshot already
// let's skip it as we alrady processed it's data from REST accounts snapshot
// let's skip it as we already processed it's data from REST accounts snapshot
logger_1.logger.log('warn', 'Ignoring WS account notification', { market: this._options.marketName });

@@ -464,0 +464,0 @@ return;

2

dist/serum_producer.d.ts

@@ -16,3 +16,3 @@ import { MessageType } from './consts';

type: MessageType;
symbol: string;
market: string;
publish: boolean;

@@ -19,0 +19,0 @@ payload: string;

6

dist/types.d.ts

@@ -18,7 +18,7 @@ import { Op, Channel, MessageType } from './consts';

readonly type: 'recent_trades';
readonly symbol: string;
readonly market: string;
readonly trades: Trade[];
}
export interface DataMessage extends Message {
readonly symbol: string;
readonly market: string;
readonly version: number;

@@ -106,3 +106,3 @@ readonly slot: number;

export declare type SerumListMarketItem = {
symbol: string;
name: string;
address: string;

@@ -109,0 +109,0 @@ baseCurrency: string;

2

package.json
{
"name": "serum-vial",
"version": "0.7.1",
"version": "0.8.0",
"engines": {

@@ -5,0 +5,0 @@ "node": ">=15"

365

README.md
<img src="https://raw.githubusercontent.com/tardis-dev/serum-vial/master/logo.svg">
# serum-vial: real-time market data server for Serum DEX
# serum-vial: real-time WS market data API for Serum
[![Version](https://img.shields.io/npm/v/serum-vial.svg)](https://www.npmjs.org/package/serum-vial)
[![Version](https://img.shields.io/npm/v/serum-vial.svg?color=05aac5)](https://www.npmjs.org/package/serum-vial)
[![Docker version](https://img.shields.io/docker/v/tardisdev/serum-vial/latest?label=Docker&color=05aac5)](https://hub.docker.com/r/tardisdev/serum-vial)
<br/>
## Why?
We all know that Serum DEX is awesome, but since it's a new ecosystem, some tooling around it may not be so convenient and productive especially from centralized exchanges APIs users perspective. Serum-vial which is a real-time WebSocket market data API server for Serum DEX hopes to alleviate some of those issues by offering:
- **familiar experience for centralized exchanges APIs users**
- **WebSocket API with Pub/Sub flow** - subscribe to selected channels and markets and receive real-time data as easy to parse JSON messages that can be consumed from any language supporting WebSocket protocol
- **incremental L2 order book updates** - instead of decoding Serum market `asks` and `bids` accounts for each account change in order to detect order book changes, receive initial L2 snapshot and incremental updates as JSON messages real-time over WebSocket connection
- **tick-by-tick trades** - instead of decoding `eventQueue` account data which can be large (>1MB) and in practice it's hard to consume real-time directly from Solana RPC node due to it's size, receive individual trade messages real-time over WebSocket connection
- **real-time L3 data** - receive the most granular updates on individual order level, opens, changes, fills and cancellations for each order Serum DEX handles
- **decreased load and bandwidth consumption for Solana RPC nodes hosts** - by providing real-time market data API via serum-vial server instead of RPC node directly, hosts can decrease substantially both CPU load and bandwidth requirements as only serum-vial will be direct consumer of RPC API when it comes to market data accounts changes and will efficiently normalize and broadcast small JSON messages to all connected clients
<br/>
<br/>
## What about placing/cancelling orders endpoints?
Serum-vial provides real-time market data only and does not include endpoints for placing/canceling or tracking own orders as that requires handling private keys which is currently out of scope of this project.
Both [serum-rest-server](https://github.com/project-serum/serum-rest-server) and [@project-serum/serum](https://github.com/project-serum/serum-ts/tree/master/packages/serum) provide such functionality and are recommended alternatives.
<br/>
<br/>
## Getting started
Run the code snippet below in the browser Dev Tools directly or in Node.js (requires installation of `ws` lib, [see](https://runkit.com/thad/serum-vial-node-js-sample)).
```js
// connect to hosted demo server
const ws = new WebSocket('wss://serum-vial.tardis.dev/v1/ws')
// if connecting to serum-vial server running locally
// const ws = new WebSocket('ws://localhost:8000/v1/ws')
ws.onmessage = (message) => {
console.log(JSON.parse(message.data))
}
ws.onopen = () => {
// subscribe both to L2 and L3 real-time channels
const subscribeL2 = {
op: 'subscribe',
channel: 'level2',
markets: ['BTC/USDC']
}
const subscribeL3 = {
op: 'subscribe',
channel: 'level3',
markets: ['BTC/USDC']
}
ws.send(JSON.stringify(subscribeL2))
ws.send(JSON.stringify(subscribeL3))
}
```
[![Try this code live on RunKit](https://img.shields.io/badge/-Try%20this%20code%20live%20on%20RunKit-c?color=05aac5)](https://runkit.com/thad/serum-vial-node-js-sample)
<br/>
<br/>
## Demo
Demo of Serum DEX UI backed by serum-vial WebSocket API for trade and order book data is available at:
[serum-dex.tardis.dev](https://serum-dex.tardis.dev/)
Since by default serum-vial uses [`confirmed` commitment level](https://docs.solana.com/developing/clients/jsonrpc-api#configuring-state-commitment) for getting accounts notification from RPC node, it may sometimes feel slightly lagging when it comes to order book updates vs default DEX UI which uses [`recent/processed` commitment](https://docs.solana.com/developing/clients/jsonrpc-api#configuring-state-commitment). Trade data is provided faster since by default DEX UI is pooling `eventQueue` account data on interval due to it's size (> 1MB), and serum-vial uses real-time `eventQueue` account notification as a source for trade messages which aren't delayed by pooling interval time.
[![See demo](https://img.shields.io/badge/-See%20Demo-c?color=05aac5)](https://serum-dex.tardis.dev/)
<br/>
<br/>
## Installation
- ### npx <sub>(requires Node.js >= 15 installed on host machine)</sub>
### npx <sub>(requires Node.js >= 15 and git installed on host machine)</sub>
That will start serum-vial server running on port `8000`
Installs and starts serum-vial server running on port `8000`.
```sh
npx serum-vial
```
```sh
npx serum-vial
```
If you'd like to switch to different Serum Node endpoint, change port or run with debug logs enabled, just add one of the available CLI options:
If you'd like to switch to different Solana RPC node endpoint, change port or run with debug logs enabled, just add one of the available CLI options.
```sh
npx serum-vial --endpoint https://solana-api.projectserum.com --log-level debug --port 8080
```
```sh
npx serum-vial --endpoint https://solana-api.projectserum.com --log-level debug --port 8080
```
Run `npx serum-vial --help` to see all available startup options (node endpoint url, port etc.)
<br/>
<br/>
Alternatively you can install serum-vial globally.
- ### npm <sub>(requires Node.js >= 12 installed on host machine)</sub>
```sh
npm install -g serum-vial
serum-vial
```
Installs `serum-vial` globally and runs it on port `8000`.
<br/>
```sh
npm install -g serum-vial
serum-vial
```
#### CLI options
If you'd like to switch to different Serum Node endpoint, change port or run with debug logs enabled, just add one of the available CLI options:
| &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; name &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; | default | description |
| ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------- |
| `port` | 8000 | Port to bind server on |
| `endpoint` | https://solana-api.projectserum.com | Solana RPC node endpoint that serum-vial uses as a data source |
| `log-level` | info | Log level, available options: debug, info, warn and error |
| `minions-count` | 1 | Minions worker threads count that are responsible for broadcasting normalized WS messages to connected clients |
| `commitment` | confirmed | Solana commitment level to use when communicating with RPC node, available options: confirmed and processed |
| `markets-json` | `@project-serum/serum` [markets.json](https://github.com/project-serum/serum-ts/blob/master/packages/serum/src/markets.json) file, but only non depreciated markets | path to custom market.json definition file if one wants to run serum-vial for custom markets |
```sh
serum-vial --endpoint https://solana-api.projectserum.com --debug --port 8080
```
<br/>
Run `npx serum-vial --help` to see all available startup options.
<br/>
<br/>
Run `serum-vial --help` to see all available startup options (node endpoint url, port etc.)
<br/>
<br/>
### Docker
- ### Docker
Pulls and runs latest version of [`tardisdev/serum-vial` image](https://hub.docker.com/r/tardisdev/serum-vial). Serum Matchine server will available on host via `8000` port (for example [http://localhost:8000/v1/markets](http://localhost:8000/v1/markets)) with debug logs enabled (`SV_LOG_LEVEL` env var).
```sh
docker run -p 8000:8000 -e "SM_ENDPOINT=https://solana-api.projectserum.com" -e "SV_LOG_LEVEL=debug" -d tardisdev/serum-vial:latest
```
<br/>
<br/>
Pulls and runs latest version of [`tardisdev/serum-vial` Docker Image](https://hub.docker.com/r/tardisdev/serum-vial) on port `8000`.
## Architecture
```sh
docker run -p 8000:8000 -d tardisdev/serum-vial:latest
```
![architecture diagram](https://user-images.githubusercontent.com/51779538/111766810-3f20e080-88a6-11eb-8c4c-54787332cc84.png)
If you'd like to switch to different Solana RPC node endpoint, change port or run with debug logs enabled, just specify those via one of the available env variables.
- server runs with multiple\* `Minions` worker threads and multiple `Serum Producers`
- `Minions` are responsible for WebSockets subscriptions management, constructing L2 & L1 messages out of L3 messages published by `Serum Producer` and broadcasting all those messages to all subscribed clients
- `Serum Producer` is responsible for connecting to Serum Node RPC WS API and subscribing all relevant accounts changes (event & request queue, bids & asks) for all supported markets as well as producing L3 market data messages that are then passed to minions and published as WebSocket messages to all subscribed clients
- by default all non depreciated markets, can be changed by providing market.json
```sh
docker run -p 8000:8000 -e "SV_LOG_LEVEL=debug" -d tardisdev/serum-vial:latest
```
\* multi core support via [`worker_threads`](https://nodejs.org/api/worker_threads.html) for `Minions` is linux only feature which allows multiple threads to bind to the same port, see https://github.com/uNetworking/uWebSockets.js/issues/304 and https://lwn.net/Articles/542629/ - for other OSes there's only one worker thread running
<br/>
#### ENV Variables
| &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; name &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; | default | description |
| ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------- |
| `SV_PORT` | 8000 | Port to bind server on |
| `SV_ENDPOINT` | https://solana-api.projectserum.com | Solana RPC node endpoint that serum-vial uses as a data source |
| `SV_LOG_LEVEL` | info | Log level, available options: debug, info, warn and error |
| `SV_MINIONS_COUNT` | 1 | Minions worker threads count that are responsible for broadcasting normalized WS messages to connected clients |
| `SV_COMMITMENT` | confirmed | Solana commitment level to use when communicating with RPC node, available options: confirmed and processed |
| `SV_MARKETS_JSON` | `@project-serum/serum` [markets.json](https://github.com/project-serum/serum-ts/blob/master/packages/serum/src/markets.json) file, but only non depreciated markets | path to custom market.json definition file if one wants to run serum-vial for custom markets |
<br/>
<br/>
## WebSocket `/ws` endpoint
### SSL/TLS Support
Allows subscribing to Serum DEX real-market data streams.
Serum-vial supports [SSL/TLS](https://en.wikipedia.org/wiki/Transport_Layer_Security) but it's not enabled by default. In order to enable it you need to set `CERT_FILE_NAME` env var pointing to the certificate file and `KEY_FILE_NAME` pointing to private key of that certificate.
<br/>
<br/>
## WebSocket API
WebSocket API provides real-time market data feeds of Serum DEX and uses a bidirectional protocol which encodes all messages as JSON objects.
Every message has a `type` field that is determining it's data type so it can be handled appropriately.
Each WebSocket client is required to actively send native WebSocket [pings](https://developer.mozilla.org/en-US/docs/Web/API/WebSockets_API/Writing_WebSocket_servers#pings_and_pongs_the_heartbeat_of_websockets) to the server with interval less than 30 seconds, otherwise connection may be dropped due to inactivity.
All messages timestamps are returned in [ISO 8601](https://en.wikipedia.org/wiki/ISO_8601) format, for example: `"2021-03-23T17:03:03.994Z"`.
<br/>
### Endpoint URL
#### `ws://localhost:8000/v1/ws`
(assuming serum-vial runs locally on default port without SSL enabled)
<br/>
### Subscribing to data feeds
To begin receiving real-time market data feed messages, you must first send a subscribe message to the server indicating which channels and markets to receive.
If you want to unsubscribe from channel and markets, send an unsubscribe message. The structure is equivalent to subscribe messages except `op` field which should be set to `"op": "unsubscribe"`.
```js
const ws = new WebSocket('ws://localhost:8000/v1/ws')
ws.onmessage = (message) => {
console.log(message)
}
ws.onopen = () => {
const subscribePayload = {
const subscribeL2 = {
op: 'subscribe',
channel: 'level2', // or level1, level3, trades
channel: 'level2',
markets: ['BTC/USDC']
}
ws.send(JSON.stringify(subscribePayload))
ws.send(JSON.stringify(subscribeL2))
}

@@ -92,8 +208,146 @@ ```

<br/>
#### Subscribe/unsubscribe message format
```ts
{
"op": "subscribe" | "unsubscribe",
"channel": "level3" | "level2" | "level1" | "trades",
"markets": string[]
}
```
##### Sample `subscribe` message
```json
{
"op": "subscribe",
"channel": "level2",
"markets": ["BTC/USDC"]
}
```
<br/>
## HTTP endpoints
#### Subscription confirmation message format
### `/markets`
Once a subscribe (or unsubscribe) message is received by the server, it will respond with a `subscribed` (or `unsubscribed`) confirmation message or `error` if received message was invalid.
```ts
{
"type": "subscribed" | "unsubscribed",
"channel": "level3" | "level2" | "level1" | "trades",
"markets": string[],
"timestamp": string
}
```
##### Sample `subscribed` confirmation message
```json
{
"type": "subscribed",
"channel": "level2",
"markets": ["BTC/USDC"],
"timestamp": "2021-03-23T17:06:30.010Z"
}
```
<br/>
#### Error message format
Error message is returned for invalid subscribe/unsubscribe messages - no existing market, invalid channel name etc.
```ts
{
"type": "error",
"message": "string,
"timestamp": "string
}
```
##### Sample `error` message
```json
{
"type": "error",
"message": "Invalid channel provided: 'levels1'.",
"timestamp": "2021-03-23T17:13:31.010Z"
}
```
<br/>
<br/>
### Available channels & corresponding message types
TODO: Subscribing to channel results in providing messages with various types
- `trades`
- `recent_trades`
- `trade`
- `level1`
- `recent_trades`
- `trade`
- `quote`
- `level2`
- `l2snapshot`
- `l2update`
- `recent_trades`
- `trade`
- `level3`
- `l3snapshot`
- `open`
- `fill`
- `change`
- `done`
<br/>
<br/>
### Available markets
<br/>
<br/>
### Data messages
#### `recent_trades`
TODO: returned from oldest to newest
```ts
{
"type": "recent_trades",
"market": string,
"trades": Trade[],
"timestamp": string
}
```
#### Sample `recent_trades` message
<br/>
#### `trade`
```ts
```
<br/>
<br/>
## HTTP API
### GET `/markets`
Accepts no params and returns non depreciated Serum markets.

@@ -130,1 +384,8 @@

```
<br/>
<br/>
## Architecture
![architecture diagram](https://user-images.githubusercontent.com/51779538/112196249-20567d00-8c0b-11eb-86c9-409c1de75c41.png)

10

src/boot_server.ts

@@ -19,3 +19,3 @@ import os from 'os'

const MINIONS_COUNT = os.platform() === 'linux' ? minionsCount : 1
let readyMonionsCount = 0
let readyMinionsCount = 0

@@ -26,3 +26,3 @@ logger.log(

)
minionReadyChannel.onmessage = () => readyMonionsCount++
minionReadyChannel.onmessage = () => readyMinionsCount++

@@ -37,3 +37,3 @@ // start minions workers and wait until all are ready

minionWorker.on('error', (err) => {
logger.log('error', `Minion worker ${minionWorker.threadId} error occured: ${err.message} ${err.stack}`)
logger.log('error', `Minion worker ${minionWorker.threadId} error occurred: ${err.message} ${err.stack}`)
throw err

@@ -48,3 +48,3 @@ })

while (true) {
if (readyMonionsCount === MINIONS_COUNT) {
if (readyMinionsCount === MINIONS_COUNT) {
break

@@ -72,3 +72,3 @@ }

'error',
`Serum producer worker ${serumProducerWorker.threadId} error occured: ${err.message} ${err.stack}`
`Serum producer worker ${serumProducerWorker.threadId} error occurred: ${err.message} ${err.stack}`
)

@@ -75,0 +75,0 @@ throw err

34

src/data_mapper.ts

@@ -31,3 +31,3 @@ import { EVENT_QUEUE_LAYOUT, Market, Orderbook, getLayoutVersion } from '@project-serum/serum'

private _bidsAccountSlabItems: SlabItem[] | undefined = undefined
private _asksAccounSlabItems: SlabItem[] | undefined = undefined
private _asksAccountSlabItems: SlabItem[] | undefined = undefined

@@ -113,3 +113,3 @@ // _local* are used only for verification purposes

this._asksAccounSlabItems = newAsksSlabItems
this._asksAccountSlabItems = newAsksSlabItems
this._asksAccountOrders = newAsksOrders

@@ -154,3 +154,3 @@ }

type: 'l3snapshot',
symbol: this._options.symbol,
market: this._options.symbol,
timestamp,

@@ -175,3 +175,3 @@ slot,

this._currentL2Snapshot = {
asks: this._mapToL2Snapshot(this._asksAccounSlabItems!),
asks: this._mapToL2Snapshot(this._asksAccountSlabItems!),
bids: this._mapToL2Snapshot(this._bidsAccountSlabItems!)

@@ -182,3 +182,3 @@ }

type: 'l2snapshot',
symbol: this._options.symbol,
market: this._options.symbol,
timestamp,

@@ -198,3 +198,3 @@ slot,

type: 'quote',
symbol: this._options.symbol,
market: this._options.symbol,
timestamp,

@@ -216,3 +216,3 @@ slot,

accountsData.asks !== undefined
? this._mapToL2Snapshot(this._asksAccounSlabItems!)
? this._mapToL2Snapshot(this._asksAccountSlabItems!)
: this._currentL2Snapshot.asks,

@@ -247,3 +247,3 @@

type: 'trade',
symbol: this._options.symbol,
market: this._options.symbol,
timestamp,

@@ -264,3 +264,3 @@ slot,

type: 'recent_trades',
symbol: this._options.symbol,
market: this._options.symbol,
timestamp,

@@ -282,3 +282,3 @@ trades: [...this._recentTrades.items()]

type: 'l2snapshot',
symbol: this._options.symbol,
market: this._options.symbol,
timestamp,

@@ -292,3 +292,3 @@ slot,

type: 'l2update',
symbol: this._options.symbol,
market: this._options.symbol,
timestamp,

@@ -315,3 +315,3 @@ slot,

type: 'quote',
symbol: this._options.symbol,
market: this._options.symbol,
timestamp,

@@ -505,3 +505,3 @@ slot,

}
// remove from currrent levels map so we know that such level exists in new levels
// remove from current levels map so we know that such level exists in new levels
currentLevelsMap.delete(newLevel[0])

@@ -559,3 +559,3 @@ } else {

type: message.type,
symbol: message.symbol,
market: message.market,
publish,

@@ -590,3 +590,3 @@ payload: JSON.stringify(message),

type: 'fill',
symbol: this._options.symbol,
market: this._options.symbol,
timestamp,

@@ -614,3 +614,3 @@ slot,

type: 'done',
symbol: this._options.symbol,
market: this._options.symbol,
timestamp,

@@ -696,3 +696,3 @@ slot,

type,
symbol: this._options.symbol,
market: this._options.symbol,
timestamp,

@@ -699,0 +699,0 @@ slot,

2

src/index.ts
export { bootServer, stopServer } from './boot_server'
export { getDefaultMarkets } from './helpers'
export { logger } from './logger'
export { getDefaultMarkets } from './helpers'
export * from './types'

1

src/logger.ts
import winston from 'winston'
const { combine, timestamp, printf, colorize, uncolorize } = winston.format

@@ -3,0 +4,0 @@

30

src/minion.ts

@@ -1,11 +0,11 @@

import { Market, getLayoutVersion } from '@project-serum/serum'
import { getLayoutVersion, Market } from '@project-serum/serum'
import { Connection, PublicKey } from '@solana/web3.js'
import {
App,
SSLApp,
HttpResponse,
SHARED_COMPRESSOR,
SSLApp,
TemplatedApp,
WebSocket,
us_listen_socket_close
us_listen_socket_close,
WebSocket
} from 'uWebSockets.js'

@@ -73,6 +73,6 @@ import { isMainThread, threadId, workerData } from 'worker_threads'

private readonly _l2SnapshotsSerialized: { [symbol: string]: string } = {}
private readonly _l3SnapshotsSerialized: { [symbol: string]: string } = {}
private readonly _recentTradesSerialized: { [symbol: string]: string } = {}
private readonly _quotesSerialized: { [symbol: string]: string } = {}
private readonly _l2SnapshotsSerialized: { [market: string]: string } = {}
private readonly _l3SnapshotsSerialized: { [market: string]: string } = {}
private readonly _recentTradesSerialized: { [market: string]: string } = {}
private readonly _quotesSerialized: { [market: string]: string } = {}
private readonly _marketNames: string[]

@@ -153,3 +153,3 @@ private _listenSocket: any | undefined = undefined

const serumMarket: SerumListMarketItem = {
symbol: market.name,
name: market.name,
baseCurrency: baseCurrency!,

@@ -186,3 +186,3 @@ quoteCurrency: quoteCurrency!,

public async processMessage(message: MessageEnvelope) {
const topic = `${message.type}/${message.symbol}`
const topic = `${message.type}/${message.market}`

@@ -194,14 +194,14 @@ if (logger.level === 'debug') {

if (message.type === 'l2snapshot') {
this._l2SnapshotsSerialized[message.symbol] = message.payload
this._l2SnapshotsSerialized[message.market] = message.payload
}
if (message.type === 'l3snapshot') {
this._l3SnapshotsSerialized[message.symbol] = message.payload
this._l3SnapshotsSerialized[message.market] = message.payload
}
if (message.type === 'quote') {
this._quotesSerialized[message.symbol] = message.payload
this._quotesSerialized[message.market] = message.payload
}
if (message.type === 'recent_trades') {
this._recentTradesSerialized[message.symbol] = message.payload
this._recentTradesSerialized[message.market] = message.payload
}

@@ -277,3 +277,3 @@

type: 'recent_trades',
symbol: market,
market,
timestamp: new Date().toISOString(),

@@ -280,0 +280,0 @@ trades: []

10

src/rpc_client.ts
import { Market } from '@project-serum/serum'
import { AccountInfo, Commitment, PublicKey } from '@solana/web3.js'
import AbortController from 'abort-controller'
import fetch from 'node-fetch'
import AbortController from 'abort-controller'
import { PassThrough } from 'stream'

@@ -127,3 +127,3 @@ import WebSocket from 'ws'

// this helper class handles RPC subscriptions to seprate DEX accounts (bids, asks & event queue)
// this helper class handles RPC subscriptions to separate DEX accounts (bids, asks & event queue)
// and provide notification in synchronized fashion, meaning we get at most one notification per slot

@@ -234,3 +234,3 @@ // with accounts data that changed in that slot

logger.log('info', 'Estabilished new RPC WebSocket connection...', { market: this._options.marketName })
logger.log('info', 'Established new RPC WebSocket connection...', { market: this._options.marketName })
}

@@ -296,3 +296,3 @@

if (message.method === 'slotNotification') {
// ignore slot notficiations which are only used as a heartbeat message
// ignore slot notifications which are only used as a heartbeat message
return

@@ -593,3 +593,3 @@ }

// in case we fetched accounts data via REST API and WS account notification is published for such snapshot already
// let's skip it as we alrady processed it's data from REST accounts snapshot
// let's skip it as we already processed it's data from REST accounts snapshot
logger.log('warn', 'Ignoring WS account notification', { market: this._options.marketName })

@@ -596,0 +596,0 @@ return

2

src/serum_producer.ts

@@ -90,3 +90,3 @@ import { Market } from '@project-serum/serum'

type: MessageType
symbol: string
market: string
publish: boolean

@@ -93,0 +93,0 @@ payload: string

6

src/types.ts

@@ -22,3 +22,3 @@ import { Op, Channel, MessageType } from './consts'

readonly type: 'recent_trades'
readonly symbol: string
readonly market: string
readonly trades: Trade[]

@@ -28,3 +28,3 @@ }

export interface DataMessage extends Message {
readonly symbol: string
readonly market: string
readonly version: number

@@ -126,3 +126,3 @@ readonly slot: number

export type SerumListMarketItem = {
symbol: string
name: string
address: string

@@ -129,0 +129,0 @@ baseCurrency: string

dist/boot_server.js.map

Sorry, the diff of this file is not supported yet

dist/data_mapper.d.ts.map

Sorry, the diff of this file is not supported yet

dist/data_mapper.js.map

Sorry, the diff of this file is not supported yet

dist/index.d.ts.map

Sorry, the diff of this file is not supported yet

dist/index.js.map

Sorry, the diff of this file is not supported yet

dist/logger.d.ts.map

Sorry, the diff of this file is not supported yet

dist/logger.js.map

Sorry, the diff of this file is not supported yet

dist/minion.js.map

Sorry, the diff of this file is not supported yet

dist/rpc_client.js.map

Sorry, the diff of this file is not supported yet

dist/types.d.ts.map

Sorry, the diff of this file is not supported yet

SocketSocket SOC 2 Logo

Product

  • Package Alerts
  • Integrations
  • Docs
  • Pricing
  • FAQ
  • Roadmap
  • Changelog

About

Packages

npm

Go

Maven

PyPI

Rubygems

Stay in touch

Get open source security insights delivered straight into your inbox.

  • Terms
  • Privacy
  • Security

Made with ⚡️ by Socket Inc