flops-utils - pypi Package Compare versions

Comparing version 0.6.0 to 0.6.1

flops_utils/logging.py

@@ -0,1 +1,3 @@
+# NOTE: This logging file provides a common logger utility for all FLOps components.
+# This avoid the risk of creating divergent duplicates of loggers in different places.
 # Reference:
@@ -2,0 +4,0 @@ # https://alexandra-zaharia.github.io/posts/make-your-own-custom-color-formatter-with-python-logging/

flops_utils/ml_repo_building_blocks/__init__.py

@@ -0,1 +1,2 @@
+# flake8: noqa: F401
 from .flops_learner_files_proxy import load_dataset

flops_utils/ml_rep...ing_blocks/flops_learner_files_proxy.py

 # Note: This proxy is used to provide ML repo developers/users with stub FLOps Learner components.
 # E.g. The ML repo developer does not have access to any data of the worker nodes yet.
-# This data will be fetched by the Learner's data_loading from the Data Manager Sidecar.
+# This data will be fetched by the Learner's data_loading from the Data Manager Sidecar/ML-Server.
 # This data_loading is part of the Learner image and should be abstracted away from the ML repo.
@@ -11,3 +11,2 @@ # To be able to include the data_loading methods in the ML repo code these mocks are provided.
 import datasets  # type: ignore
-
 from flops_utils.logging import logger
@@ -14,0 +13,0 @@

flops_utils/ml_repo_files_proxy.py

@@ -1,2 +0,2 @@
-# Note: This proxy is used to handle ml repo files
+# Note: This proxy is used to handle ML repository files
 # which get injected during the image build.
@@ -17,3 +17,3 @@ # I.e. these files are not yet present.
     except ImportError:
-        logger.exception("A ML repo file was not found.")
+        logger.exception("An ML repository file was not found.")
         sys.exit(1)

flops_utils/ml_repo_templates.py

@@ -10,2 +10,3 @@ from abc import ABC, abstractmethod
         """Calls the load_ml_data function and does data preprocessing, etc. (optional)
+
         The Learner does not yet have the data from the worker node.
@@ -22,2 +23,3 @@ To get this data please use the 'load_ml_data' function which can be imported like this:
         """Get the necessary data for training and evaluation.
+
         This data has to be already prepared/preprocessed.
@@ -37,2 +39,3 @@
         """Gets the data from the DataManager and makes it available to the model.
+
         Do not include this method call in the ModelManager init function.
@@ -43,3 +46,3 @@ The aggregator also needs the model but does not have access to the data.
 
-        Heavily depends on the underlying model and ML library.
+        It heavily depends on the underlying model and ML library.
 
@@ -80,5 +83,6 @@ Examples: ()
         - self.model.set_weights(parameters)
+
         - params_dict = zip(self.model.state_dict().keys(), parameters)
-          state_dict = OrderedDict({k: torch.tensor(v) for k, v in params_dict})
-          self.model.load_state_dict(state_dict, strict=True)
+        - state_dict = OrderedDict({k: torch.tensor(v) for k, v in params_dict})
+        - self.model.load_state_dict(state_dict, strict=True)
         """
@@ -85,0 +89,0 @@ pass

flops_utils/mqtt_topics.py

@@ -42,2 +42,6 @@ import dataclasses
 class SupportedTopic(CustomEnum):
+    """
+    This enum class represent all MQTT topics that the FLOps Manager supports.
+    """
+
     PROJECT_OBSERVER_FAILED = str(
@@ -44,0 +48,0 @@ Topic(subject=Subject.PROJECT_OBSERVER, status=Status.FAILED)

flops_utils/notifications.py

@@ -6,3 +6,2 @@ import json
 import paho.mqtt.client as paho_mqtt
-
 from flops_utils.logging import logger
@@ -9,0 +8,0 @@ from flops_utils.mqtt_topics import SupportedTopic

flops_utils/timer.py

@@ -43,2 +43,7 @@ from dataclasses import dataclass, field
 class Timer:
+    """
+    This custom timer can be used to easily record how long specific parts of code execution take.
+    It is currently only used by the image builder to track the duration of individual steps.
+    """
+
     time_stamps: Dict[str, datetime] = field(default_factory=dict)
@@ -45,0 +50,0 @@ time_frames: Dict[str, TimeFrame] = field(default_factory=dict)

flops_utils/types.py

@@ -9,6 +9,8 @@ import enum
 
-# These flavors are a subset of MLflow's model flavors.
-# They are used to decide which MLflow model flavor to use.
-# https://mlflow.org/docs/latest/models.html#built-in-model-flavors
 class MLModelFlavor(str, enum.Enum):
+    """These flavors are a subset of MLflow's model flavors.
+    They are used to decide which MLflow model flavor to use.
+    https://mlflow.org/docs/latest/models.html#built-in-model-flavors
+    """
+
     SKLEARN = "sklearn"  # Scikit-learn
@@ -18,3 +20,3 @@ PYTORCH = "pytorch"
     KERAS = "keras"
-    # This list can be further expanded.
+    # This list can be further expanded to allow further libraries & frameworks.
 
@@ -21,0 +23,0 @@

PKG-INFO

 Metadata-Version: 2.1
 Name: flops_utils
-Version: 0.6.0
-Summary: A library containing commong utilities for FLOps
+Version: 0.6.1
+Summary: A library containing common utilities for FLOps
 Author: Alexander Malyuk
@@ -15,3 +15,95 @@ Author-email: [e-mail]
 
-# TODO
+# flops-utils
+This package/library contains common pieces of code for the [FLOps project](https://github.com/oakestra/addon-FLOps?tab=readme-ov-file).
 
+
+## User Facing Features
+
+flops-utils provides FLOPs users with guiding abstract template classes for implementing their ML models and "proxy/place-holder" components to satisfy linters.
+
+When FLOps users want to structure their ML code to match the structural requirements of FLOps they should do the following:
+
+```Python
+# data_manager.py
+
+from flops_utils.ml_repo_building_blocks import load_dataset
+from flops_utils.ml_repo_templates import DataManagerTemplate
+
+class DataManager(DataManagerTemplate):
+    def __init__(self):
+        ... = self._prepare_data()
+
+    def _prepare_data(self, partition_id=1) -> Any:
+        dataset = load_dataset()
+        ...
+        return ...
+
+    def get_data(self) -> Tuple[Any, Any]:
+        return ...
+
+```
+
+```Python
+# model_manager.py
+
+import warnings
+from typing import Any, Tuple
+
+import numpy as np
+from data_manager import DataManager
+from flops_utils.ml_repo_templates import ModelManagerTemplate
+from sklearn.linear_model import LogisticRegression
+from sklearn.metrics import log_loss
+
+
+class ModelManager(ModelManagerTemplate):
+    def __init__(self):
+        self.model = ...
+        self._set_init_params()
+
+    def _set_init_params(self) -> None:
+        ...
+
+    def set_model_data(self) -> None:
+        ... = DataManager().get_data()
+
+    def get_model(self) -> Any:
+        return self.model
+
+    def get_model_parameters(self) -> Any:
+        params = ...
+        return params
+
+    def set_model_parameters(self, parameters) -> None:
+        ...
+
+    def fit_model(self) -> int:
+        return len(self.x_train)
+
+    def evaluate_model(self) -> Tuple[Any, Any, int]:
+        loss = ...
+        accuracy = ...
+        return loss, accuracy, len(self.x_test)
+        
+```
+
+A tangible example implementation is available [here](https://github.com/Malyuk-A/flops_ml_repo_mnist_sklearn/tree/main).
+
+## Internal FLOps Commonalities
+- Logger
+- Types
+- MQTT Topics
+- Handlers
+    - Notifications (Project Observer & FLOps Manager)
+    - Environment Variables
+- Auxiliary Constructs
+    - Running Shell Commands
+    - Measuring Execution Time Frames
+
+## Installation Alternatives
+
+One can also include and use these utilities by doing the following:
+```
+pip install --no-cache-dir git+https://github.com/oakestra/addon-FLOps.git@main#subdirectory=utils_library
+```
+

pyproject.toml

 [tool.poetry]
 name = "flops_utils"
-version = "0.6.0"
-description = "A library containing commong utilities for FLOps"
+version = "0.6.1"
+description = "A library containing common utilities for FLOps"
 authors = ["Alexander Malyuk <[e-mail]>"]
@@ -6,0 +6,0 @@ readme = "README.md"

README.md

@@ -1,1 +0,93 @@
-# TODO
+# flops-utils
+This package/library contains common pieces of code for the [FLOps project](https://github.com/oakestra/addon-FLOps?tab=readme-ov-file).
+
+
+## User Facing Features
+
+flops-utils provides FLOPs users with guiding abstract template classes for implementing their ML models and "proxy/place-holder" components to satisfy linters.
+
+When FLOps users want to structure their ML code to match the structural requirements of FLOps they should do the following:
+
+```Python
+# data_manager.py
+
+from flops_utils.ml_repo_building_blocks import load_dataset
+from flops_utils.ml_repo_templates import DataManagerTemplate
+
+class DataManager(DataManagerTemplate):
+    def __init__(self):
+        ... = self._prepare_data()
+
+    def _prepare_data(self, partition_id=1) -> Any:
+        dataset = load_dataset()
+        ...
+        return ...
+
+    def get_data(self) -> Tuple[Any, Any]:
+        return ...
+
+```
+
+```Python
+# model_manager.py
+
+import warnings
+from typing import Any, Tuple
+
+import numpy as np
+from data_manager import DataManager
+from flops_utils.ml_repo_templates import ModelManagerTemplate
+from sklearn.linear_model import LogisticRegression
+from sklearn.metrics import log_loss
+
+
+class ModelManager(ModelManagerTemplate):
+    def __init__(self):
+        self.model = ...
+        self._set_init_params()
+
+    def _set_init_params(self) -> None:
+        ...
+
+    def set_model_data(self) -> None:
+        ... = DataManager().get_data()
+
+    def get_model(self) -> Any:
+        return self.model
+
+    def get_model_parameters(self) -> Any:
+        params = ...
+        return params
+
+    def set_model_parameters(self, parameters) -> None:
+        ...
+
+    def fit_model(self) -> int:
+        return len(self.x_train)
+
+    def evaluate_model(self) -> Tuple[Any, Any, int]:
+        loss = ...
+        accuracy = ...
+        return loss, accuracy, len(self.x_test)
+        
+```
+
+A tangible example implementation is available [here](https://github.com/Malyuk-A/flops_ml_repo_mnist_sklearn/tree/main).
+
+## Internal FLOps Commonalities
+- Logger
+- Types
+- MQTT Topics
+- Handlers
+    - Notifications (Project Observer & FLOps Manager)
+    - Environment Variables
+- Auxiliary Constructs
+    - Running Shell Commands
+    - Measuring Execution Time Frames
+
+## Installation Alternatives
+
+One can also include and use these utilities by doing the following:
+```
+pip install --no-cache-dir git+https://github.com/oakestra/addon-FLOps.git@main#subdirectory=utils_library
+```

Alert delta unavailable

Currently unable to show alert delta for PyPI packages.

Improved metrics

Total package byte prevSize

21656

34.12%

Lines of code

390

2.9%