scriptflow - pypi Package Compare versions

Comparing version 0.2.9 to 0.2.10

PKG-INFO

@@ -1,10 +0,9 @@
-Metadata-Version: 2.3
+Metadata-Version: 2.4
 Name: scriptflow
-Version: 0.2.9
+Version: 0.2.10
 Summary: Like a makefile but in python, a stripped-down system of Airflow or Luigi
 Author: Thibaut Lamadon
 Author-email: [e-mail]
-Requires-Python: >=3.7,<4.0
+Requires-Python: >=3.8,<4.0
 Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.7
 Classifier: Programming Language :: Python :: 3.8
@@ -16,2 +15,3 @@ Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
 Provides-Extra: dask
@@ -21,5 +21,4 @@ Requires-Dist: asyncssh (>=2.9.0,<3.0.0)
 Requires-Dist: omegaconf (>=2.1.1,<3.0.0)
-Requires-Dist: pytest (>=7.1.1,<8.0.0)
 Requires-Dist: requests (>=2.27.1,<3.0.0)
-Requires-Dist: rich (>=11.0.0,<12.0.0)
+Requires-Dist: rich (>=14.0.0)
 Requires-Dist: tinydb (>=4.7.0,<5.0.0)
@@ -132,3 +131,3 @@ Requires-Dist: toml (>=0.10.2,<0.11.0)
       outputs = f"final.txt",
-      inputs = [*t1.get_outputs(),*t1.get_outputs()])
+      inputs = [*task1.get_outputs(),*task2.get_outputs()])
 
@@ -144,3 +143,3 @@ await task_final
 pip install scriptflow
-scritpflow run sleepit
+scriptflow run sleepit
 ```
@@ -147,0 +146,0 @@

pyproject.toml

 [tool.poetry]
 name = "scriptflow"
-version = "0.2.9"
+version = "0.2.10"
 description = "Like a makefile but in python, a stripped-down system of Airflow or Luigi"
@@ -10,4 +10,4 @@ authors = ["Thibaut Lamadon <[e-mail]>"]
 [tool.poetry.dependencies]
-python = "^3.7"
-rich = "^11.0.0"
+python = "^3.8"
+rich = ">=14.0.0"
 toml = "^0.10.2"
@@ -18,3 +18,2 @@ click = "^8.0.3"
 tinydb = "^4.7.0"
-pytest = "^7.1.1"
 omegaconf = "^2.1.1"
@@ -28,2 +27,3 @@
 codecov = "^2.1.12"
+pytest = "^7.1.1"
 
@@ -30,0 +30,0 @@ [tool.poetry.scripts]

README.md

@@ -103,3 +103,3 @@ # scriptflow
       outputs = f"final.txt",
-      inputs = [*t1.get_outputs(),*t1.get_outputs()])
+      inputs = [*task1.get_outputs(),*task2.get_outputs()])
 
@@ -115,3 +115,3 @@ await task_final
 pip install scriptflow
-scritpflow run sleepit
+scriptflow run sleepit
 ```
@@ -118,0 +118,0 @@

scriptflow/cli.py

@@ -29,2 +29,7 @@ import click
 
+def handle_exc(loop, context):
+    msg = context.get("message", "Unhandled exception")
+    exc = context.get("exception")
+    print(f"[loop {id(loop)}] {msg}: {exc!r}")
+
 """
@@ -35,2 +40,7 @@ Main
 
+    print("Running flow: {}".format(func.__name__))
+
+    loop = asyncio.get_running_loop()
+    loop.set_exception_handler(handle_exc)
+
     task_controller = asyncio.create_task( controller.start_loops() )
@@ -41,3 +51,4 @@ task_controller.add_done_callback(_handle_task_result)
 
-    await func()      
+    await func()
+    # await task_controller # wait for the controller to finish      
     
@@ -44,0 +55,0 @@ # let the other guys finish

scriptflow/core.py

@@ -194,9 +194,12 @@ # simple script flow
             if UP_TO_DATE:
-                self.log(f"task [red]{task.uid}[/red] is up to date, skipping it.")
+                self.log(f"task [green]{task.uid}[/green] is up to date, skipping it.")
                 return(True)
         else:
-            self.log(f"adding [red]{task.uid}[/red] output {task.outputs} is missing")
+            self.log(f"running [red]{task.uid}[/red] because output is missing: {task.outputs} ")
 
         self.log(f"adding [red]{task.uid}[/red]")
-        self.log(" - cmd: {}".format( " ".join(task.get_command() ) ))
+        if task.shell:
+            self.log(" > cmd: {}".format( task.get_command() ) )
+        else:
+            self.log(" > cmd: {}".format( " ".join(task.get_command() ) ))
         return(False)
@@ -203,0 +206,0 @@

scriptflow/runners.py

@@ -20,2 +20,3 @@ """
 import time
+from .task import Task
 
@@ -25,11 +26,11 @@ class AbstractRunner(ABC):
     @abstractmethod
-    def size():
+    def size(self) -> int:
         pass
 
     @abstractmethod
-    def available_slots():
+    def available_slots(self):
         pass
 
     @abstractmethod
-    def add(self, task):
+    def add(self, task:Task):
         pass
@@ -54,12 +55,17 @@
     """
-    def add(self, task):
+    def add(self, task:Task):
 
         if task.quiet:
+            print(f"Running task: {task.get_command()} with shell={task.shell} (quiet)")
             subp = subprocess.Popen(task.get_command(),
                 stdout=subprocess.DEVNULL,
-                stderr=subprocess.STDOUT)
+                stderr=subprocess.STDOUT,
+                shell=task.shell)
         else:
-            subp = subprocess.Popen(task.get_command())
-                #stdout=subprocess.DEVNULL,
-                #stderr=subprocess.STDOUT)
+            print("Running task (not quiet): {}".format(task.get_command()))
+            log_file = open(f"{task.uid}.log", "w")
+            subp = subprocess.Popen(task.get_command(),
+                stdout=log_file,
+                stderr=subprocess.STDOUT,
+                shell=task.shell)
 
@@ -77,3 +83,4 @@ self.processes[task.hash] = {
         while True:
-            self.update(controller)                
+            self.update(controller)   
+            # print(f"Running loop, checking processes... {self.size()} processes running.")             
             await asyncio.sleep(0.1)
@@ -84,2 +91,3 @@
         for (k,p) in self.processes.items():
+            # print(f"Process {k} completed, removing from runner.")
             poll_val = p["proc"].poll()
@@ -90,2 +98,3 @@ if poll_val is not None:
         for k in to_remove:
+            # print(f"Process {k} completed, removing from runner.")
             task = self.processes[k]["task"]
@@ -92,0 +101,0 @@ del self.processes[k]

scriptflow/task.py

@@ -30,7 +30,9 @@
 class Task:
-    cmd =""
-    uid=""
+    cmd=""
+    uid:str=""
     mem="1"
     ncore="1"
+    shell:bool=False
     retry=0
+    quiet:bool=True
 
@@ -45,5 +47,10 @@ """
 
+        if "shell" in kwargs.keys():
+            self.shell  = kwargs["shell"]
+        else:
+            self.props = {}
+
         if "cmd" in kwargs.keys():
-            # we check if we have a string, in which case we try to split it
-            if isinstance(kwargs["cmd"], str):
+            # we check if we have a string, in which case we try to split it, but only if shell is False
+            if (not self.shell) and (isinstance(kwargs["cmd"], str)):
                 self.cmd = shlex.split(kwargs["cmd"])
@@ -98,2 +105,4 @@ else:
 
+
+
     def __await__(self):
@@ -100,0 +109,0 @@ # we need to check if the task has been scheduled

Alert delta unavailable

Currently unable to show alert delta for PyPI packages.

Improved metrics

Total package byte prevSize

41038

3.09%

Lines of code

689

3.92%