order-book - npm Package Compare versions

Comparing version 0.4.2 to 0.4.3

CHANGES.md

 ## Changelog
 
+### 0.4.3 (2022-05-29)
+ * Bugfix: handle scientific notation of small values in Kraken checksum
+ * Update: calculate Kraken checksum on order books less than 10 levels deep
+ * Bugfix: fix occasional incorrect checksums for OKX, FTX and Bitget
+
 ### 0.4.2 (2022-04-17)
@@ -4,0 +9,0 @@ * Update: OKEx renamed OKX (for checksum validation)

order_book.egg-info/PKG-INFO

 Metadata-Version: 2.1
 Name: order-book
-Version: 0.4.2
+Version: 0.4.3
 Summary: A fast orderbook implementation, in C, for Python
@@ -147,2 +147,7 @@ Home-page: https://github.com/bmoscon/orderbook
 
+### 0.4.3 (2022-05-29)
+ * Bugfix: handle scientific notation of small values in Kraken checksum
+ * Update: calculate Kraken checksum on order books less than 10 levels deep
+ * Bugfix: fix occasional incorrect checksums for OKX, FTX and Bitget
+
 ### 0.4.2 (2022-04-17)
@@ -149,0 +154,0 @@ * Update: OKEx renamed OKX (for checksum validation)

orderbook/orderbook.c

@@ -10,3 +10,5 @@ /*
 
+typedef int (*string_builder_t)(PyObject *pydata, uint8_t *data, int *pos);
 
+
 void Orderbook_dealloc(Orderbook *self)
@@ -181,3 +183,3 @@ {
 {
-	return SortedDict_len(self->bids) + SortedDict_len(self->asks);
+    return SortedDict_len(self->bids) + SortedDict_len(self->asks);
 }
@@ -273,2 +275,4 @@
     PyObject *m;
+    OrderBookModuleState *st;
+
     if (PyType_Ready(&OrderbookType) < 0 || PyType_Ready(&SortedDictType) < 0)
@@ -295,2 +299,27 @@ return NULL;
 
+    st = get_order_book_state(m);
+
+    PyObject* builtins = PyImport_AddModule("builtins");
+    if (builtins == NULL) {
+        Py_DECREF(&SortedDictType);
+        Py_DECREF(m);
+        return NULL;
+    }
+
+    st->format = PyObject_GetAttrString(builtins, "format");
+    Py_DECREF(builtins);
+    if (st->format == NULL) {
+        Py_DECREF(&SortedDictType);
+        Py_DECREF(m);
+        return NULL;
+    }
+
+    st->formatf = PyUnicode_FromString("f");
+    if (st->formatf == NULL) {
+        Py_DECREF(st->format);
+        Py_DECREF(&SortedDictType);
+        Py_DECREF(m);
+        return NULL;
+    }
+
     return m;
@@ -300,2 +329,36 @@ }
 
+static int order_book_traverse(PyObject *m, visitproc visit, void *arg)
+{
+    OrderBookModuleState* st = get_order_book_state(m);
+    Py_VISIT(st->format);
+    Py_VISIT(st->formatf);
+    return 0;
+}
+
+
+static int order_book_clear(PyObject* m)
+{
+    OrderBookModuleState* st = get_order_book_state(m);
+    Py_CLEAR(st->format);
+    Py_CLEAR(st->formatf);
+    return 0;
+}
+
+
+static void order_book_free(PyObject *m)
+{
+    order_book_clear(m);
+}
+
+
+static OrderBookModuleState* get_order_book_state(PyObject *m)
+{
+    if (m == NULL) {
+        return (OrderBookModuleState*) PyModule_GetState(PyState_FindModule(&orderbookmodule));
+    } else {
+        return (OrderBookModuleState*) PyModule_GetState(m);
+    }
+}
+
+
 // Checksums Code
@@ -324,2 +387,5 @@ static int kraken_string_builder(PyObject *pydata, uint8_t *data, int *pos)
         if (*string != '.') {
+            if (*string == 'E' || *string == 'e') {
+                break;
+            }
             if (*string != '0' && leading_zero) {
@@ -345,3 +411,8 @@ leading_zero = false;
 {
-    for(int i = 0; i < 10; ++i) { // 10 is the kraken defined number of price/size pairs to use from each side
+    uint32_t size = SortedDict_len(side);
+    if (size > 10) { // 10 is the kraken defined number of price/size pairs to use from each side
+        size = 10;
+    }
+
+    for(uint32_t i = 0; i < size; ++i) {
         PyObject *price = PyTuple_GET_ITEM(side->keys, i);
@@ -366,14 +437,6 @@ PyObject *size = PyDict_GetItem(side->data, price);
     if (EXPECT(ob->max_depth && ob->max_depth < 10, 0)) {
-        PyErr_SetString(PyExc_ValueError, "Max depth is less than minimum number of levels for Kraken checksum");
+        PyErr_SetString(PyExc_ValueError, "Max depth is less than usual number of levels for Kraken checksum");
         return NULL;
     }
 
-    uint32_t bids_size = SortedDict_len(ob->bids);
-    uint32_t asks_size = SortedDict_len(ob->asks);
-
-    if (EXPECT(bids_size < 10 || asks_size < 10, 0)) {
-        PyErr_SetString(PyExc_ValueError, "Depth is less than minimum number of levels for Kraken checksum");
-        return NULL;
-    }
-
     int pos = 0;
@@ -389,2 +452,3 @@ if (EXPECT(kraken_populate_side(ob->asks, ob->checksum_buffer, &pos), 0)) {
     unsigned long ret = crc32_table(ob->checksum_buffer, pos);
+
     return PyLong_FromUnsignedLong(ret);
@@ -394,3 +458,5 @@ }
 
-static int ftx_string_builder(PyObject *pydata, uint8_t *data, int *pos)
+
+
+static int str_string_builder(PyObject *pydata, uint8_t *data, int *pos)
 {
@@ -417,3 +483,2 @@ PyObject *repr = PyObject_Str(pydata);
     *pos += len;
-    data[(*pos)++] = ':';
 
@@ -425,4 +490,91 @@ Py_DECREF(str);
 
-static PyObject* ftx_checksum(const Orderbook *ob, const uint32_t depth)
+
+static int floatstr_string_builder(PyObject *pydata, uint8_t *data, int *pos)
 {
+    PyObject *repr = PyObject_Str(pydata);
+    if (EXPECT(!repr, 0)) {
+        return -1;
+    }
+
+    PyObject *flt = PyFloat_FromString(repr);
+    if (EXPECT(str_string_builder(flt, data, pos), 0)) {
+        Py_DECREF(flt);
+        return -1;
+    }
+
+    Py_DECREF(flt);
+    return 0;
+}
+
+
+static int formatf_string_builder(PyObject *pydata, uint8_t *data, int *pos)
+{
+    OrderBookModuleState* st = get_order_book_state(NULL);
+
+    PyObject* repr = PyObject_CallFunctionObjArgs(st->format, pydata, st->formatf, NULL);
+    if (EXPECT(!repr, 0)) {
+        return -1;
+    }
+
+    PyObject* str = PyUnicode_AsEncodedString(repr, "UTF-8", "strict");
+    Py_DECREF(repr);
+    if (EXPECT(!str, 0)) {
+        return -1;
+    }
+
+    const char *string = PyBytes_AS_STRING(str);
+    if (EXPECT(!string, 0)) {
+        Py_DECREF(str);
+        return -1;
+    }
+
+    int len = strlen(string);
+    memcpy(&data[*pos], string, len);
+    *pos += len;
+
+    Py_DECREF(str);
+    return 0;
+}
+
+
+static int okx_string_builder(PyObject *pydata, uint8_t *data, int *pos)
+{
+    int startpos = *pos;
+    if (EXPECT(str_string_builder(pydata, data, pos), 0)) {
+        return -1;
+    }
+
+    // default 'str' formatting is wrong when the value is in scientific notation
+    if (EXPECT(memchr(&data[startpos], (char) 'E', *pos - startpos), 0)) {
+        *pos = startpos;
+        if (EXPECT(formatf_string_builder(pydata, data, pos), 0)) {
+            return -1;
+        }
+    }
+
+    return 0;
+}
+
+
+static int ftx_string_builder(PyObject *pydata, uint8_t *data, int *pos)
+{
+    int startpos = *pos;
+    if (EXPECT(str_string_builder(pydata, data, pos), 0)) {
+        return -1;
+    }
+
+    // default 'str' formatting is wrong when the value is less than 0.0001 or in scientific notation
+    if (EXPECT(!strncmp(&data[startpos], "0.0000", 6) || memchr(&data[startpos], (char) 'E', *pos - startpos), 0)) {
+        *pos = startpos;
+        if (EXPECT(floatstr_string_builder(pydata, data, pos), 0)) {
+            return -1;
+        }
+    }
+
+    return 0;
+}
+
+
+static PyObject* alternating_checksum(const Orderbook *ob, const uint32_t depth, char separator, string_builder_t string_builder)
+{
     if (EXPECT(ob->max_depth && ob->max_depth < depth, 0)) {
@@ -439,3 +591,3 @@ PyErr_SetString(PyExc_ValueError, "Max depth is less than minimum number of levels for checksum");
 
-    for(uint32_t i = 0; i < depth; ++i) { // 100 is the FTX defined number of price/size pairs to use from each side, 25 is OKX/OKCOIN
+    for(uint32_t i = 0; i < depth; ++i) { 
         if (i < bids_size) {
@@ -445,9 +597,11 @@ price = PyTuple_GET_ITEM(ob->bids->keys, i);
 
-            if (EXPECT(ftx_string_builder(price, ob->checksum_buffer, &pos), 0)) {
+            if (EXPECT(string_builder(price, ob->checksum_buffer, &pos), 0)) {
                 return NULL;
             }
+            ob->checksum_buffer[pos++] = separator;
 
-            if (EXPECT(ftx_string_builder(size, ob->checksum_buffer, &pos), 0)) {
+            if (EXPECT(string_builder(size, ob->checksum_buffer, &pos), 0)) {
                 return NULL;
             }
+            ob->checksum_buffer[pos++] = separator;
         }
@@ -459,9 +613,11 @@
 
-            if (EXPECT(ftx_string_builder(price, ob->checksum_buffer, &pos), 0)) {
+            if (EXPECT(string_builder(price, ob->checksum_buffer, &pos), 0)) {
                 return NULL;
             }
+            ob->checksum_buffer[pos++] = separator;
 
-            if (EXPECT(ftx_string_builder(size, ob->checksum_buffer, &pos), 0)) {
+            if (EXPECT(string_builder(size, ob->checksum_buffer, &pos), 0)) {
                 return NULL;
             }
+            ob->checksum_buffer[pos++] = separator;
         }
@@ -482,7 +638,7 @@ }
         case FTX:
-            return ftx_checksum(ob, 100);
+            return alternating_checksum(ob, 100, ':', ftx_string_builder);
         case OKX:
-            return ftx_checksum(ob, 25);
+            return alternating_checksum(ob, 25, ':', okx_string_builder);
         case BITGET:
-            return ftx_checksum(ob, 25);
+            return alternating_checksum(ob, 25, ':', str_string_builder);
         default:
@@ -489,0 +645,0 @@ return NULL;

orderbook/orderbook.h

@@ -77,4 +77,4 @@ /*
 static PyMappingMethods Orderbook_mapping = {
-	(lenfunc)Orderbook_len,
-	(binaryfunc)Orderbook_getitem,
+    (lenfunc)Orderbook_len,
+    (binaryfunc)Orderbook_getitem,
     (objobjargproc)Orderbook_setitem
@@ -102,3 +102,15 @@ };
 
+// the module contains reusable python objects referring to the builtin format 
+// function and a fixed string 'f'
+typedef struct {
+    PyObject *format;
+    PyObject *formatf;
+} OrderBookModuleState;
 
+static int order_book_traverse(PyObject *m, visitproc visit, void *arg);
+static int order_book_clear(PyObject* m);
+static void order_book_free(PyObject *m);
+static OrderBookModuleState* get_order_book_state(PyObject *m);
+
+
 // Module specific definitions and initilization
@@ -109,3 +121,8 @@ static PyModuleDef orderbookmodule = {
     .m_doc = "Orderbook data structure",
-    .m_size = -1,
+    .m_size = sizeof(OrderBookModuleState),
+    .m_methods = NULL,
+    .m_slots = NULL,
+    .m_traverse = order_book_traverse,
+    .m_clear = order_book_clear,
+    .m_free = order_book_free,
 };
@@ -112,0 +129,0 @@

orderbook/sorteddict.c

@@ -382,3 +382,3 @@ /*
 /* Sorted Dictionary Mapping Functions */
-Py_ssize_t SortedDict_len(SortedDict *self)
+Py_ssize_t SortedDict_len(const SortedDict *self)
 {
@@ -385,0 +385,0 @@ int len = PyDict_Size(self->data);

orderbook/sorteddict.h

@@ -47,3 +47,3 @@ /*
 
-Py_ssize_t SortedDict_len(SortedDict *self);
+Py_ssize_t SortedDict_len(const SortedDict *self);
 PyObject *SortedDict_getitem(SortedDict *self, PyObject *key);
@@ -50,0 +50,0 @@ int SortedDict_setitem(SortedDict *self, PyObject *key, PyObject *value);

PKG-INFO

 Metadata-Version: 2.1
 Name: order_book
-Version: 0.4.2
+Version: 0.4.3
 Summary: A fast orderbook implementation, in C, for Python
@@ -147,2 +147,7 @@ Home-page: https://github.com/bmoscon/orderbook
 
+### 0.4.3 (2022-05-29)
+ * Bugfix: handle scientific notation of small values in Kraken checksum
+ * Update: calculate Kraken checksum on order books less than 10 levels deep
+ * Bugfix: fix occasional incorrect checksums for OKX, FTX and Bitget
+
 ### 0.4.2 (2022-04-17)
@@ -149,0 +154,0 @@ * Update: OKEx renamed OKX (for checksum validation)

setup.py

@@ -38,3 +38,3 @@ '''
     name='order_book',
-    version='0.4.2',
+    version='0.4.3',
     author="Bryant Moscon",
@@ -41,0 +41,0 @@ author_email="[e-mail]",

Alert delta unavailable

Currently unable to show alert delta for PyPI packages.

Improved metrics

Total package byte prevSize

99806

5.26%