unidist - pypi Package Compare versions

Comparing version 0.6.0 to 0.7.0

unidist/core/backends/mpi/core/memory/_memory.pyx

+# Copyright (C) 2021-2023 Modin authors
+#
+# SPDX-License-Identifier: Apache-2.0
+
+from libc.stdint cimport uint8_t, int64_t
+
+cimport memory
+
+def parallel_memcopy(const uint8_t[:] src, uint8_t[:] dst, int memcopy_threads):
+    """
+    Multithreaded data copying between buffers.
+
+    Parameters
+    ----------
+    src : uint8_t[:]
+        Copied data.
+    dst : uint8_t[:]
+        Buffer for writing.
+    memcopy_threads : int
+        Number of threads to write.
+    """
+    with nogil:
+        memory.parallel_memcopy(&dst[0],
+                                &src[0],
+                                len(src),
+                                64,
+                                memcopy_threads)
+
+def fill(int64_t[:] buff, int64_t value):
+    """
+    Fill a given buffer with a given value.
+
+    Parameters
+    ----------
+    buff : int64_t[:]
+        Original data.
+    value : int64_t
+        Value to fill.
+    """
+    with nogil:
+        memory.fill(&buff[0], len(buff), value)

unidist/core/backends/mpi/core/memory/memory.pxd

+# Copyright (C) 2021-2023 Modin authors
+#
+# SPDX-License-Identifier: Apache-2.0
+
+from libc.stdint cimport uint8_t, uintptr_t, int64_t
+
+cdef extern from "memory.cpp" nogil:
+    pass
+
+# Declare the class with cdef
+cdef extern from "memory.h" namespace "unidist" nogil:
+    void parallel_memcopy(uint8_t* dst, 
+                            const uint8_t* src, 
+                            int64_t nbytes, 
+                            uintptr_t block_size, 
+                            int num_threads)
+
+    void fill(int64_t *buff, int64_t size, int64_t value)
+

MANIFEST.in

 include versioneer.py
 include unidist/_version.py
+include unidist/core/backends/mpi/core/memory/_memory.pyx
+include unidist/core/backends/mpi/core/memory/memory.pxd

PKG-INFO

 Metadata-Version: 2.1
 Name: unidist
-Version: 0.6.0
+Version: 0.7.0
 Summary: Unified Distributed Execution
 Home-page: https://github.com/modin-project/unidist
 License: Apache-2.0
-Requires-Python: >=3.7.1
+Requires-Python: >=3.9
 Description-Content-Type: text/markdown
-License-File: LICENSE
-License-File: AUTHORS
-Requires-Dist: packaging
-Requires-Dist: cloudpickle
 Provides-Extra: ray
-Requires-Dist: ray[default]>=1.13.0; extra == "ray"
-Requires-Dist: pydantic<2; extra == "ray"
 Provides-Extra: dask
-Requires-Dist: dask[complete]>=2.22.0; extra == "dask"
-Requires-Dist: distributed>=2.22.0; extra == "dask"
 Provides-Extra: mpi
-Requires-Dist: mpi4py>=3.0.3; extra == "mpi"
-Requires-Dist: msgpack>=1.0.0; extra == "mpi"
-Requires-Dist: psutil; extra == "mpi"
 Provides-Extra: all
-Requires-Dist: ray[default]>=1.13.0; extra == "all"
-Requires-Dist: pydantic<2; extra == "all"
-Requires-Dist: dask[complete]>=2.22.0; extra == "all"
-Requires-Dist: distributed>=2.22.0; extra == "all"
-Requires-Dist: mpi4py>=3.0.3; extra == "all"
-Requires-Dist: msgpack>=1.0.0; extra == "all"
-Requires-Dist: psutil; extra == "all"
+License-File: LICENSE
+License-File: AUTHORS
 
@@ -32,0 +16,0 @@ <p align="center">

setup.py

 import pathlib
 from setuptools import setup, find_packages, Extension
-from setuptools.dist import Distribution
 from Cython.Build import cythonize
@@ -9,33 +8,2 @@ import sys
 
-try:
-    from wheel.bdist_wheel import bdist_wheel
-
-    HAS_WHEEL = True
-except ImportError:
-    HAS_WHEEL = False
-
-if HAS_WHEEL:
-
-    class UnidistWheel(bdist_wheel):
-        def finalize_options(self):
-            bdist_wheel.finalize_options(self)
-            self.root_is_pure = False
-
-        def get_tag(self):
-            _, _, plat = bdist_wheel.get_tag(self)
-            py = "py3"
-            abi = "none"
-            return py, abi, plat
-
-
-class UnidistDistribution(Distribution):
-    def __init__(self, *attrs):
-        Distribution.__init__(self, *attrs)
-        if HAS_WHEEL:
-            self.cmdclass["bdist_wheel"] = UnidistWheel
-
-    def is_pure(self):
-        return False
-
-
 # https://github.com/modin-project/unidist/issues/324
@@ -58,2 +26,4 @@ ray_deps = ["ray[default]>=1.13.0", "pydantic<2"]
     language="c++",
+    extra_compile_args=["-std=c++11"],
+    extra_link_args=["-std=c++11"],
 )
@@ -65,3 +35,2 @@
     cmdclass=versioneer.get_cmdclass(),
-    distclass=UnidistDistribution,
     description="Unified Distributed Execution",
@@ -81,4 +50,4 @@ long_description=long_description,
     },
-    python_requires=">=3.7.1",
+    python_requires=">=3.9",
     ext_modules=cythonize([_memory]),
 )

unidist.egg-info/PKG-INFO

 Metadata-Version: 2.1
 Name: unidist
-Version: 0.6.0
+Version: 0.7.0
 Summary: Unified Distributed Execution
 Home-page: https://github.com/modin-project/unidist
 License: Apache-2.0
-Requires-Python: >=3.7.1
+Requires-Python: >=3.9
 Description-Content-Type: text/markdown
-License-File: LICENSE
-License-File: AUTHORS
-Requires-Dist: packaging
-Requires-Dist: cloudpickle
 Provides-Extra: ray
-Requires-Dist: ray[default]>=1.13.0; extra == "ray"
-Requires-Dist: pydantic<2; extra == "ray"
 Provides-Extra: dask
-Requires-Dist: dask[complete]>=2.22.0; extra == "dask"
-Requires-Dist: distributed>=2.22.0; extra == "dask"
 Provides-Extra: mpi
-Requires-Dist: mpi4py>=3.0.3; extra == "mpi"
-Requires-Dist: msgpack>=1.0.0; extra == "mpi"
-Requires-Dist: psutil; extra == "mpi"
 Provides-Extra: all
-Requires-Dist: ray[default]>=1.13.0; extra == "all"
-Requires-Dist: pydantic<2; extra == "all"
-Requires-Dist: dask[complete]>=2.22.0; extra == "all"
-Requires-Dist: distributed>=2.22.0; extra == "all"
-Requires-Dist: mpi4py>=3.0.3; extra == "all"
-Requires-Dist: msgpack>=1.0.0; extra == "all"
-Requires-Dist: psutil; extra == "all"
+License-File: LICENSE
+License-File: AUTHORS
 
@@ -32,0 +16,0 @@ <p align="center">

unidist.egg-info/SOURCES.txt

@@ -56,4 +56,4 @@ AUTHORS
 unidist/core/backends/mpi/core/memory/_memory.cpp
-unidist/core/backends/mpi/core/memory/memory.cpp
-unidist/core/backends/mpi/core/memory/memory.h
+unidist/core/backends/mpi/core/memory/_memory.pyx
+unidist/core/backends/mpi/core/memory/memory.pxd
 unidist/core/backends/mpi/core/monitor/__init__.py
@@ -60,0 +60,0 @@ unidist/core/backends/mpi/core/monitor/loop.py

unidist/__init__.py

@@ -20,3 +20,3 @@ # Copyright (C) 2021-2023 Modin authors
 )
-from ._version import get_versions
+from . import _version
 
@@ -37,3 +37,2 @@ __all__ = [
 
-__version__ = get_versions()["version"]
-del get_versions
+__version__ = _version.get_versions()["version"]

unidist/_version.py

 
-# This file was generated by 'versioneer.py' (0.21) from
+# This file was generated by 'versioneer.py' (0.29) from
 # revision-control system data, or from the parent directory name of an
@@ -11,7 +11,7 @@ # unpacked source archive. Distribution tarballs contain a pre-generated copy
 {
- "date": "2024-01-08T21:52:33+0100",
+ "date": "2024-05-29T15:58:01+0200",
  "dirty": false,
  "error": null,
- "full-revisionid": "4fa3cf262299c08b66efd538c190b360c448d467",
- "version": "0.6.0"
+ "full-revisionid": "1e2922ac2cf2b707b1a0ed06c4842da5ca3ecea8",
+ "version": "0.7.0"
 }
@@ -18,0 +18,0 @@ '''  # END VERSION_JSON

unidist/core/backe...i/core/monitor/shared_memory_manager.py

@@ -211,9 +211,11 @@ # Copyright (C) 2021-2023 Modin authors
             [
-                1
-                if data_id in self._reservation_info
-                and self.shared_store.get_ref_number(
-                    data_id, self._reservation_info[data_id]["service_index"]
+                (
+                    1
+                    if data_id in self._reservation_info
+                    and self.shared_store.get_ref_number(
+                        data_id, self._reservation_info[data_id]["service_index"]
+                    )
+                    > 0
+                    else 0
                 )
-                > 0
-                else 0
                 for data_id in cleanup_list
@@ -220,0 +222,0 @@ ],

unidist/core/backends/mpi/core/shared_object_store.py

@@ -233,5 +233,7 @@ # Copyright (C) 2021-2023 Modin authors
         self.win = MPI.Win.Allocate_shared(
-            self.shared_memory_size * MPI.BYTE.size
-            if mpi_state.is_monitor_process()
-            else 0,
+            (
+                self.shared_memory_size * MPI.BYTE.size
+                if mpi_state.is_monitor_process()
+                else 0
+            ),
             MPI.BYTE.size,
@@ -249,5 +251,7 @@ comm=mpi_state.host_comm,
         self.service_win = MPI.Win.Allocate_shared(
-            self.service_info_max_count * MPI.LONG.size
-            if mpi_state.is_monitor_process()
-            else 0,
+            (
+                self.service_info_max_count * MPI.LONG.size
+                if mpi_state.is_monitor_process()
+                else 0
+            ),
             MPI.LONG.size,
@@ -368,9 +372,9 @@ comm=mpi_state.host_comm,
         with WinLock(self.service_win):
-            self.service_shared_buffer[
-                service_index + self.FIRST_DATA_INDEX
-            ] = first_index
+            self.service_shared_buffer[service_index + self.FIRST_DATA_INDEX] = (
+                first_index
+            )
             self.service_shared_buffer[service_index + self.REFERENCES_NUMBER] = 1
-            self.service_shared_buffer[
-                service_index + self.DATA_NUMBER_INDEX
-            ] = data_number
+            self.service_shared_buffer[service_index + self.DATA_NUMBER_INDEX] = (
+                data_number
+            )
             self.service_shared_buffer[service_index + self.WORKER_ID_INDEX] = worker_id
@@ -377,0 +381,0 @@

unidist/core/backends/mpi/core/memory/memory.cpp

-/* 
- * Copyright (C) 2021-2023 Modin authors
- *
- * SPDX-License-Identifier: Apache-2.0
- */
-
-#include "memory.h"
-
-#include <cstring>
-#include <thread>
-#include <vector>
-
-namespace unidist {
-
-  uint8_t *pointer_logical_and(const uint8_t *address, uintptr_t bits) {
-    uintptr_t value = reinterpret_cast<uintptr_t>(address);
-    return reinterpret_cast<uint8_t *>(value & bits);
-  }
-
-  void fill(int64_t *buff, int64_t size, int64_t value){
-    std::fill(buff, buff+size, value);
-  }
-
-  void parallel_memcopy(uint8_t *dst,
-                        const uint8_t *src,
-                        int64_t nbytes,
-                        uintptr_t block_size,
-                        int num_threads) {
-    std::vector<std::thread> threadpool(num_threads);
-    uint8_t *left = pointer_logical_and(src + block_size - 1, ~(block_size - 1));
-    uint8_t *right = pointer_logical_and(src + nbytes, ~(block_size - 1));
-    int64_t num_blocks = (right - left) / block_size;
-
-    // Update right address
-    right = right - (num_blocks % num_threads) * block_size;
-
-    // Now we divide these blocks between available threads. The remainder is
-    // handled on the main thread.
-    int64_t chunk_size = (right - left) / num_threads;
-    int64_t prefix = left - src;
-    int64_t suffix = src + nbytes - right;
-    // Now the data layout is | prefix | k * num_threads * block_size | suffix |.
-    // We have chunk_size = k * block_size, therefore the data layout is
-    // | prefix | num_threads * chunk_size | suffix |.
-    // Each thread gets a "chunk" of k blocks.
-
-    // Start all threads first and handle leftovers while threads run.
-    for (int i = 0; i < num_threads; i++) {
-      threadpool[i] = std::thread(
-          memcpy, dst + prefix + i * chunk_size, left + i * chunk_size, chunk_size);
-    }
-
-    std::memcpy(dst, src, prefix);
-    std::memcpy(dst + prefix + num_threads * chunk_size, right, suffix);
-
-    for (auto &t : threadpool) {
-      if (t.joinable()) {
-        t.join();
-      }
-    }
-  }
-
-}  // namespace unidist

unidist/core/backends/mpi/core/memory/memory.h

-/* 
- * Copyright (C) 2021-2023 Modin authors
- *
- * SPDX-License-Identifier: Apache-2.0
- */
-
-#ifndef MEMORY_H
-#define MEMORY_H
-
-#include <stdint.h>
-
-namespace unidist {
-    // A helper function for doing memcpy with multiple threads. This is required
-    // to saturate the memory bandwidth of modern cpus.
-    void parallel_memcopy(uint8_t *dst,
-                        const uint8_t *src,
-                        int64_t nbytes,
-                        uintptr_t block_size,
-                        int num_threads);
-    
-    void fill(int64_t *buff, int64_t size, int64_t value);
-}  // namespace unidist
-
-#endif  // MEMORY_H

Alert delta unavailable

Currently unable to show alert delta for PyPI packages.

Improved metrics

Total package byte prevSize

1542527

0.27%

Lines of code

12203

1.08%