myqueue - pypi Package Compare versions

Comparing version 23.1.0 to 23.4.0

myqueue.egg-info/PKG-INFO

 Metadata-Version: 2.1
 Name: myqueue
-Version: 23.1.0
+Version: 23.4.0
 Summary: Simple job queue
@@ -5,0 +5,0 @@ Home-page: https://myqueue.readthedocs.io/

myqueue/__init__.py

@@ -12,3 +12,3 @@ """Top level module definitions.
 
-__version__ = '23.1.0'
+__version__ = '23.4.0'
 
@@ -15,0 +15,0 @@

myqueue/cli.py

@@ -8,2 +8,3 @@ from __future__ import annotations
 from time import time
+from typing import Any
 
@@ -176,3 +177,3 @@
 
-        def a(*args, **kwargs):  # type: ignore
+        def a(*args: str, **kwargs: Any) -> None:
             """Wrapper for Parser.add_argument().
@@ -252,3 +253,5 @@
               help='Extra arguments for scheudler.  Example: '
-              '-X "--gres=gpu:4".  Can be used multiple times.')
+              '-X bla-bla.  For arguments that start with a dash, '
+              'leave out the space: -X--gres=gpu:4 or -X=--gres=gpu:4. '
+              'Can be used multiple times.')
 
@@ -255,0 +258,0 @@ if cmd == 'modify':

myqueue/complete.py

@@ -13,19 +13,19 @@ #!/usr/bin/env python3
 import sys
-from typing import Any, Iterable, Mapping
+from typing import Iterable, Mapping, TYPE_CHECKING
+if TYPE_CHECKING:
+    from myqueue.task import Task
 
 
-def read() -> dict[str, Any]:
+def read() -> list[Task]:
     """Read queue as a dict."""
     from pathlib import Path
-    import json
-    from myqueue.config import find_home_folder
-    home = find_home_folder(Path('.').resolve())
-    path = home / '.myqueue/queue.json'
-    try:
-        dct: dict[str, Any] = json.loads(path.read_text())
-        return dct
-    except Exception:
-        return {}
+    from myqueue.config import Configuration
+    from myqueue.queue import Queue
+    from myqueue.selection import Selection
 
+    config = Configuration.read()
+    with Queue(config, need_lock=False) as queue:
+        return queue.select(Selection(folders=[Path('.').resolve()]))
 
+
 # Beginning of computer generated data:
@@ -108,12 +108,9 @@ commands = {
     if previous in ['-n', '--name']:
-        dct = read()
-        words = set()
-        for task in dct['tasks']:
-            cmd = task['cmd']
-            words.add((cmd['cmd'] + '+' + '_'.join(cmd['args'])).rstrip('+'))
+        tasks = read()
+        words = [task.cmd.name for task in tasks]
         return words
 
     if previous in ['-i', '--id']:
-        dct = read()
-        return {str(task['id']) for task in dct['tasks']}
+        tasks = read()
+        return {str(task.id) for task in tasks}
 
@@ -120,0 +117,0 @@ if command == 'help':

myqueue/queue.py

@@ -19,12 +19,12 @@ """Queue class for interacting with the queue.
 from types import TracebackType
-from typing import Iterator, Iterable, Sequence
-from typing_extensions import LiteralString
+from typing import Iterable, Iterator, Sequence
 
 from myqueue.config import Configuration
+from myqueue.migration import migrate
 from myqueue.schedulers import Scheduler, get_scheduler
+from myqueue.selection import Selection
 from myqueue.states import State
 from myqueue.task import Task, create_task
-from myqueue.utils import Lock, cached_property, normalize_folder
-from myqueue.selection import Selection
-from myqueue.migration import migrate
+from myqueue.utils import Lock, cached_property, normalize_folder, plural
+from typing_extensions import LiteralString
 
@@ -339,2 +339,3 @@ VERSION = 11
 
+    skipped = 0
     for task in tasks:
@@ -365,3 +366,5 @@ task.dtasks = []
                 else:
-                    raise DependencyError(f'Bad state ({state}): {name}')
+                    task.state = State.CANCELED
+                    skipped += 1
+                    continue
 
@@ -372,7 +375,11 @@ task.dtasks.append(dtask)
 
+    if skipped:
+        print(f'Skipping {plural(skipped, "task")} '
+              'because of dependency in bad state')
 
+
 def dump_db(path: Path) -> None:
     """Pretty-print content of sqlite3 db file."""
+    from rich.console import Console
     from rich.table import Table
-    from rich.console import Console
     prnt = Console().print
@@ -379,0 +386,0 @@ db = sqlite3.connect(path)

myqueue/resources.py

@@ -84,4 +84,7 @@ """Resource class to handle resource requirements: time, cores, processes."""
 
-        >>> Resources.from_string('16:1:xeon8:2h')
+        >>> r = Resources.from_string('16:1:xeon8:2h')
+        >>> r
         Resources(cores=16, processes=1, tmax=7200, nodename='xeon8')
+        >>> print(r)
+        16:1:xeon8:2h
         >>> Resources.from_string('16:1m')
@@ -148,6 +151,6 @@ Resources(cores=16, tmax=60)
         s = str(self.cores)
+        if self.processes != self.cores:
+            s += ':' + str(self.processes)
         if self.nodename:
             s += ':' + self.nodename
-        if self.processes != self.cores:
-            s += ':' + str(self.processes)
         s += ':' + seconds_to_short_time_string(self.tmax)
@@ -154,0 +157,0 @@ if self.weight > 0.0:

myqueue/selection.py

@@ -14,3 +14,3 @@ from __future__ import annotations
                  name: str | None = None,
-                 states: set[State] = set(),
+                 states: set[State] = None,
                  folders: list[Path] = [],
@@ -50,3 +50,3 @@ recursive: bool = True,
         args = []
-        if len(self.states) < 8:
+        if self.states is not None and len(self.states) < 8:
             q = ', '.join('?' * len(self.states))
@@ -53,0 +53,0 @@ parts.append(f'state IN ({q})')

myqueue/test/complete_test.py

 from __future__ import annotations
-import json
 import os
 
 from myqueue.complete import complete, main
+from myqueue.queue import Queue
+from myqueue.task import create_task
 
@@ -50,7 +51,10 @@
     mq.mkdir()
-    dct = {'tasks': [{'cmd': {'cmd': 'abc123', 'args': []}, 'id': 117}]}
-    (mq / 'queue.json').write_text(json.dumps(dct))
+    (mq / 'config.py').write_text('config = {"scheduler": "local"}\n')
+    task = create_task('abc123')
+    task.id = 117
+    with Queue() as queue:
+        queue.add(task)
     words = complete('', '-n', 'mq ls -n ', 9)
-    assert words == {'abc123'}
+    assert words == ['abc123']
     words = complete('', '-i', 'mq ls -i ', 9)
     assert words == {'117'}

myqueue/test/workflow_test.py

@@ -124,2 +124,22 @@ from __future__ import annotations
 
+wf_block = """
+from myqueue.task import task
+def create_tasks():
+    t1 = task('shell:sleep+a')
+    t2 = task('shell:echo+hello', deps=[t1], name='hello')
+    t3 = task('shell:echo+bye')
+    return [t1, t2, t3]
+"""
+
+
+def test_workflow_with_failed_job_blocking(mq):
+    """Failefd dependency (t1) for t2 should not block t3."""
+    Path('wf.py').write_text(wf_block)
+    mq('workflow wf.py . -t hello')  # submit t1 and t2
+    assert mq.wait() == 'FC'
+    mq('rm -sC . --force')
+    mq('workflow wf.py .')
+    assert mq.wait() == 'Fd'
+
+
 wf2 = """
@@ -126,0 +146,0 @@ from myqueue.task import task

PKG-INFO

 Metadata-Version: 2.1
 Name: myqueue
-Version: 23.1.0
+Version: 23.4.0
 Summary: Simple job queue
@@ -5,0 +5,0 @@ Home-page: https://myqueue.readthedocs.io/

Alert delta unavailable

Currently unable to show alert delta for PyPI packages.

Improved metrics

Total package byte prevSize

261992

0.38%

Lines of code

5786

0.49%