statsig-python-core - pypi Package Compare versions

Comparing version 0.17.2rc2603240138 to 0.17.2b2603241852

Cargo.toml

@@ -21,3 +21,3 @@ [profile.release]
 license = "ISC"
-version = "0.17.2-rc.2603240138"
+version = "0.17.2-beta.2603241852"
 homepage = "https://statsig.com/"
@@ -24,0 +24,0 @@ authors = ["Statsig", "Daniel Loomb <[e-mail]>"]

PKG-INFO

 Metadata-Version: 2.4
 Name: statsig_python_core
-Version: 0.17.2rc2603240138
+Version: 0.17.2b2603241852
 Classifier: Programming Language :: Rust
@@ -5,0 +5,0 @@ Classifier: Programming Language :: Python :: Implementation :: CPython

py_src/statsig_python_core/statsig_python_core.pyi

@@ -93,2 +93,4 @@ # This file is automatically generated by pyo3_stub_gen
     def preload_multi(self, data: typing.Sequence[bytes]) -> None: ...
+    def write_mmap_data(self, data: typing.Sequence[bytes], path: builtins.str) -> None: ...
+    def preload_mmap(self, path: builtins.str) -> None: ...
 
@@ -95,0 +97,0 @@ @typing.final

statsig-pyo3/Cargo.toml

@@ -19,3 +19,3 @@ [package]
 pyo3 = { version = "0.28", features = ["abi3-py310"] }
-pyo3-stub-gen = "0.19"
+pyo3-stub-gen = "0.20"
 serde_json = { version = "1.0.125", features = ["float_roundtrip"] }
@@ -22,0 +22,0 @@ statsig-rust = { path = "../statsig-rust", features = [

statsig-pyo3/src/interned_store_py.rs

@@ -0,1 +1,2 @@
+use pyo3::exceptions::PyRuntimeError;
 use pyo3::prelude::*;
@@ -5,2 +6,3 @@ use pyo3::pyclass;
 use pyo3::types::PyBytes;
+use pyo3::types::PyType;
 use pyo3_stub_gen::derive::*;
@@ -21,4 +23,4 @@
 impl InternedStorePy {
-    #[staticmethod]
-    pub fn preload(data: &Bound<'_, PyBytes>) {
+    #[classmethod]
+    pub fn preload(_cls: &Bound<'_, PyType>, data: &Bound<'_, PyBytes>) -> PyResult<()> {
         let bytes: &[u8] = data.as_bytes();
@@ -28,7 +30,10 @@
             log_e!(TAG, "Failed to preload interned store: {}", e);
+            return Err(PyRuntimeError::new_err(e.to_string()));
         }
+
+        Ok(())
     }
 
-    #[staticmethod]
-    pub fn preload_multi(data: Vec<Bound<'_, PyBytes>>) {
+    #[classmethod]
+    pub fn preload_multi(_cls: &Bound<'_, PyType>, data: Vec<Bound<'_, PyBytes>>) -> PyResult<()> {
         let bytes: Vec<&[u8]> = data.iter().map(|data| data.as_bytes()).collect();
@@ -38,4 +43,33 @@
             log_e!(TAG, "Failed to preload interned store: {}", e);
+            return Err(PyRuntimeError::new_err(e.to_string()));
         }
+
+        Ok(())
     }
+
+    #[classmethod]
+    pub fn write_mmap_data(
+        _cls: &Bound<'_, PyType>,
+        data: Vec<Bound<'_, PyBytes>>,
+        path: &str,
+    ) -> PyResult<()> {
+        let bytes: Vec<&[u8]> = data.iter().map(|data| data.as_bytes()).collect();
+
+        if let Err(e) = InternedStore::write_mmap_data(&bytes, path) {
+            log_e!(TAG, "Failed to preload mmap: {}", e);
+            return Err(PyRuntimeError::new_err(e.to_string()));
+        }
+
+        Ok(())
+    }
+
+    #[classmethod]
+    pub fn preload_mmap(_cls: &Bound<'_, PyType>, path: &str) -> PyResult<()> {
+        if let Err(e) = InternedStore::preload_mmap(path) {
+            log_e!(TAG, "Failed to load mmap data: {}", e);
+            return Err(PyRuntimeError::new_err(e.to_string()));
+        }
+
+        Ok(())
+    }
 }

statsig-rust/Cargo.toml

@@ -42,3 +42,3 @@ [package]
 sha2 = "0.10.8"
-sigstat-grpc = { path = "../statsig-grpc", version = "0.17.2-rc.2603240138", optional = true }
+sigstat-grpc = { path = "../statsig-grpc", version = "0.17.2-beta.2603241852", optional = true }
 simple_logger = { version = "5.0.0" }
@@ -54,2 +54,5 @@ tempfile = "3.8.1"
 prost = { version = "0.14", features = ["derive"] }
+memmap2 = "0.9"
+rkyv = "0.8"
+ouroboros = "0.18.0"
 
@@ -56,0 +59,0 @@ [target.'cfg(target_env = "gnu")'.dependencies]

statsig-rust/src/i...ts__/interned_store_returnable_tests.rs

@@ -91,2 +91,36 @@ use std::collections::HashMap;
     }
+
+    #[test]
+    fn test_preloading_mmap_across_forks() {
+        let path = "/tmp/statsig-rust-test-mmap.bin";
+        if std::fs::File::open(path).is_ok() {
+            std::fs::remove_file(path).unwrap();
+        }
+
+        assert!(InternedStore::write_mmap_data(&[EVAL_PROJ_JSON], path).is_ok());
+
+        let pid = unsafe { libc::fork() };
+        if pid == 0 {
+            let result = InternedStore::preload_mmap(path);
+            assert!(result.is_ok());
+
+            let json_res = DynamicReturnable::from_map(HashMap::from([(
+                "value".to_string(),
+                serde_json::Value::String("control".to_string()),
+            )]));
+            assert!(matches!(
+                json_res.value,
+                DynamicReturnableValue::JsonStatic(_)
+            ));
+
+            std::process::exit(0);
+        }
+
+        unsafe {
+            let mut status: i32 = 0;
+            libc::waitpid(pid, &mut status, 0);
+            assert_eq!(libc::WEXITSTATUS(status), 0);
+        };
+    }
+
 }

statsig-rust/src/i..._tests__/interned_store_string_tests.rs

@@ -99,2 +99,31 @@ use rusty_fork::rusty_fork_test;
     }
+
+    #[test]
+    fn test_preloading_mmap_across_forks() {
+        let path = "/tmp/statsig-rust-test-mmap.bin";
+        if std::fs::File::open(path).is_ok() {
+            std::fs::remove_file(path).unwrap();
+        }
+
+        assert!(InternedStore::write_mmap_data(&[EVAL_PROJ_JSON], path).is_ok());
+
+        let pid = unsafe { libc::fork() };
+        if pid == 0 {
+            let result = InternedStore::preload_mmap(path);
+            assert!(result.is_ok());
+
+            let key = InternedString::from_str_ref("userID");
+            assert!(matches!(key.value, InternedStringValue::Static(_)));
+            assert_eq!(key.as_str(), "userID");
+
+            std::process::exit(0);
+        }
+
+        unsafe {
+            let mut status: i32 = 0;
+            libc::waitpid(pid, &mut status, 0);
+            assert_eq!(libc::WEXITSTATUS(status), 0);
+        };
+    }
+
 }

statsig-rust/src/interned_values/interned_store.rs

 use std::{
     borrow::Cow,
     collections::hash_map::Entry,
+    fs::{File, OpenOptions},
+    io::Write,
     sync::{Arc, OnceLock},
@@ -10,3 +12,6 @@ time::{Duration, Instant},
 use lazy_static::lazy_static;
+use memmap2::Mmap;
+use ouroboros::self_referencing;
 use parking_lot::Mutex;
+use rkyv::{Archive, Deserialize as RkyvDeserialize, Serialize as RkyvSerialize};
 use serde_json::value::RawValue;
@@ -35,2 +40,3 @@
 static IMMORTAL_DATA: OnceLock<ImmortalData> = OnceLock::new();
+static MMAP_DATA: OnceLock<LoadedMmapData> = OnceLock::new();
 
@@ -41,2 +47,17 @@ lazy_static! {
 
+#[derive(Default, Archive, RkyvDeserialize, RkyvSerialize)]
+struct MmapData {
+    strings: std::collections::HashMap<u64, String>,
+    returnables: std::collections::HashMap<u64, String>,
+}
+
+#[self_referencing]
+struct LoadedMmapData {
+    file: File,
+    mmap: Mmap,
+
+    #[borrows(mmap)]
+    archived: &'this ArchivedMmapData,
+}
+
 /// Immortal vs Mutable Data
@@ -58,3 +79,2 @@ /// ------------------------------------------------------------
 }
-
 #[derive(Default)]
@@ -112,5 +132,60 @@ struct MutableData {
 
+    pub fn write_mmap_data(data: &[&[u8]], path: &str) -> Result<(), StatsigErr> {
+        let mut file = OpenOptions::new()
+            .read(true)
+            .write(true)
+            .create(true)
+            .truncate(true)
+            .open(path)
+            .map_err(|e| StatsigErr::FileError(e.to_string()))?;
+
+        let specs_responses = data
+            .iter()
+            .map(|data| try_parse_as_json(data).or_else(|_| try_parse_as_proto(data)))
+            .collect::<Result<Vec<SpecsResponseFull>, StatsigErr>>()?;
+
+        let mmap_data = mutable_to_mmap_data(specs_responses)?;
+        let archived =
+            rkyv::to_bytes::<rkyv::rancor::Error>(&mmap_data).expect("Failed to archive mmap data");
+
+        file.write_all(&archived)
+            .map_err(|e| StatsigErr::FileError(e.to_string()))?;
+        file.sync_all()
+            .map_err(|e| StatsigErr::FileError(e.to_string()))?;
+
+        log_d!(TAG, "Wrote {} bytes to mmap file", archived.len());
+
+        Ok(())
+    }
+
+    pub fn preload_mmap(path: &str) -> Result<(), StatsigErr> {
+        let file = File::open(path).map_err(|e| StatsigErr::FileError(e.to_string()))?;
+        let mmap = unsafe { Mmap::map(&file).map_err(|e| StatsigErr::FileError(e.to_string()))? };
+
+        let loaded_result = LoadedMmapDataTryBuilder {
+            file,
+            mmap,
+            archived_builder: |mmap| rkyv::access::<ArchivedMmapData, rkyv::rancor::Error>(mmap),
+        }
+        .try_build();
+
+        let loaded = match loaded_result {
+            Ok(loaded) => loaded,
+            Err(e) => {
+                return Err(StatsigErr::SerializationError(e.to_string()));
+            }
+        };
+
+        MMAP_DATA
+            .set(loaded)
+            .map_err(|_| StatsigErr::LockFailure("Failed to set MMAP_DATA".to_string()))
+    }
+
     pub fn get_or_intern_string<T: AsRef<str> + ToString>(value: T) -> InternedString {
         let hash = hashing::hash_one(value.as_ref().as_bytes());
 
+        if let Some(string) = get_string_from_mmap(hash) {
+            return InternedString::from_static(hash, string);
+        }
+
         if let Some(string) = get_string_from_shared(hash) {
@@ -135,2 +210,6 @@ return InternedString::from_static(hash, string);
 
+        if let Some(returnable) = get_returnable_from_mmap(hash) {
+            return DynamicReturnable::from_static(hash, returnable);
+        }
+
         if let Some(returnable) = get_returnable_from_shared(hash) {
@@ -181,2 +260,7 @@ return DynamicReturnable::from_static(hash, returnable);
         let hash = hashing::hash_one(bytes);
+
+        if let Some(returnable) = get_returnable_from_mmap(hash) {
+            return Some(DynamicReturnable::from_static(hash, returnable));
+        }
+
         if let Some(returnable) = get_returnable_from_shared(hash) {
@@ -285,2 +369,9 @@ return Some(DynamicReturnable::from_static(hash, returnable));
 
+fn get_string_from_mmap(hash: u64) -> Option<&'static str> {
+    let data = MMAP_DATA.get()?;
+    let archived_hash = rkyv::primitive::ArchivedU64::from_native(hash);
+    let found = data.borrow_archived().strings.get(&archived_hash);
+    found.map(|s| s.as_str())
+}
+
 fn get_string_from_shared(hash: u64) -> Option<&'static str> {
@@ -312,2 +403,17 @@ match IMMORTAL_DATA.get() {
 
+fn get_returnable_from_mmap(hash: u64) -> Option<&'static RawValue> {
+    let data = MMAP_DATA.get()?;
+
+    let archived_hash = rkyv::primitive::ArchivedU64::from_native(hash);
+    let found = data.borrow_archived().returnables.get(&archived_hash)?;
+
+    match serde_json::from_str(found) {
+        Ok(raw) => Some(raw),
+        Err(e) => {
+            log_e!(TAG, "Failed to parse returnable from mmap: {}", e);
+            None
+        }
+    }
+}
+
 fn get_returnable_from_shared(hash: u64) -> Option<&'static RawValue> {
@@ -455,2 +561,34 @@ match IMMORTAL_DATA.get() {
 
+fn mutable_to_mmap_data(specs_responses: Vec<SpecsResponseFull>) -> Result<MmapData, StatsigErr> {
+    let mutable_data: MutableData = {
+        let mut mutable_data_lock = MUTABLE_DATA.lock();
+        std::mem::take(&mut *mutable_data_lock)
+    };
+    let mut mmap_data = MmapData::default();
+
+    for (hash, arc) in mutable_data.strings.into_iter() {
+        let taken = arc.to_string();
+        mmap_data.strings.insert(hash, taken);
+    }
+
+    for (hash, returnable) in mutable_data.returnables.into_iter() {
+        let raw_string = returnable.get();
+        mmap_data.returnables.insert(hash, raw_string.to_string());
+    }
+
+    // TODO: Add evaluator values to mmap data
+    // for (hash, evaluator_value) in mutable_data.evaluator_values.into_iter() {
+    //     let raw_evaluator_value = Arc::into_raw(evaluator_value);
+    //     let leaked = unsafe { &*raw_evaluator_value };
+    //     mmap_data.evaluator_values.insert(hash, leaked);
+    // }
+
+    // held until after the mmap data is written to the file
+    for response in specs_responses {
+        drop(response);
+    }
+
+    Ok(mmap_data)
+}
+
 fn try_insert_specs(source: SpecsHashMap, destination: &mut AHashMap<u64, &'static Spec>) {
@@ -457,0 +595,0 @@ for (name, spec_ptr) in source.0.into_iter() {

statsig-rust/src/statsig_metadata.rs

@@ -13,3 +13,3 @@ use crate::log_e;
 
-pub const SDK_VERSION: &str = "0.17.2-rc.2603240138";
+pub const SDK_VERSION: &str = "0.17.2-beta.2603241852";
 
@@ -16,0 +16,0 @@ const TAG: &str = stringify!(StatsigMetadata);

Alert delta unavailable

Currently unable to show alert delta for PyPI packages.

Improved metrics

Total package byte prevSize

14165777

0.09%

Lines of code

5479

0.04%