task-queue - pypi Package Compare versions

Comparing version 2.11.0 to 2.12.0

PKG-INFO

 Metadata-Version: 2.1
 Name: task-queue
-Version: 2.11.0
+Version: 2.12.0
 Summary: Multithreaded cloud queue client.
@@ -74,3 +74,3 @@ Home-page: https://github.com/seung-lab/python-task-queue/
         
-        For small jobs, you might want to use one or more processes to execute the tasks:
+        For small jobs, you might want to use one or more processes to execute the tasks.
         
@@ -80,2 +80,3 @@ ```python
         from taskqueue import LocalTaskQueue
+        from mylibrary import PrintTask # mylibrary is wherever you defined PrintTask
         
@@ -107,2 +108,4 @@ tq = LocalTaskQueue(parallel=5) # use 5 processes
         
+        IMPORTANT: You must import the tasks that will be executed, otherwise the code to execute them has not been loaded.
+        
         ```python
@@ -112,2 +115,3 @@ # import gevent.monkey
         from taskqueue import TaskQueue
+        from mylibrary import PrintTask # mylibrary is wherever you defined PrintTask
         
@@ -114,0 +118,0 @@ # region is SQS specific, green means cooperative threading

README.md

@@ -66,3 +66,3 @@ [![Build Status](https://travis-ci.org/seung-lab/python-task-queue.svg?branch=master)](https://travis-ci.org/seung-lab/python-task-queue) [![PyPI version](https://badge.fury.io/py/task-queue.svg)](https://badge.fury.io/py/task-queue)
 
-For small jobs, you might want to use one or more processes to execute the tasks:
+For small jobs, you might want to use one or more processes to execute the tasks.
 
@@ -72,2 +72,3 @@ ```python
 from taskqueue import LocalTaskQueue
+from mylibrary import PrintTask # mylibrary is wherever you defined PrintTask
 
@@ -99,2 +100,4 @@ tq = LocalTaskQueue(parallel=5) # use 5 processes
 
+IMPORTANT: You must import the tasks that will be executed, otherwise the code to execute them has not been loaded.
+
 ```python
@@ -104,2 +107,3 @@ # import gevent.monkey
 from taskqueue import TaskQueue
+from mylibrary import PrintTask # mylibrary is wherever you defined PrintTask
 
@@ -106,0 +110,0 @@ # region is SQS specific, green means cooperative threading

requirements.txt

@@ -10,4 +10,4 @@ boto3
 pbr
-tenacity
+tenacity>=8.0.1
 tqdm
 requests>2,<3

task_queue.egg-info/pbr.json

@@ -1,1 +0,1 @@
-{"git_version": "f0f9506", "is_release": true}
+{"git_version": "d370e0b", "is_release": true}

task_queue.egg-info/PKG-INFO

 Metadata-Version: 2.1
 Name: task-queue
-Version: 2.11.0
+Version: 2.12.0
 Summary: Multithreaded cloud queue client.
@@ -74,3 +74,3 @@ Home-page: https://github.com/seung-lab/python-task-queue/
         
-        For small jobs, you might want to use one or more processes to execute the tasks:
+        For small jobs, you might want to use one or more processes to execute the tasks.
         
@@ -80,2 +80,3 @@ ```python
         from taskqueue import LocalTaskQueue
+        from mylibrary import PrintTask # mylibrary is wherever you defined PrintTask
         
@@ -107,2 +108,4 @@ tq = LocalTaskQueue(parallel=5) # use 5 processes
         
+        IMPORTANT: You must import the tasks that will be executed, otherwise the code to execute them has not been loaded.
+        
         ```python
@@ -112,2 +115,3 @@ # import gevent.monkey
         from taskqueue import TaskQueue
+        from mylibrary import PrintTask # mylibrary is wherever you defined PrintTask
         
@@ -114,0 +118,0 @@ # region is SQS specific, green means cooperative threading

task_queue.egg-info/requires.txt

@@ -10,4 +10,4 @@ boto3
 pbr
-tenacity
+tenacity>=8.0.1
 tqdm
 requests<3,>2

taskqueue/__init__.py

@@ -8,2 +8,2 @@ from .registered_task import RegisteredTask, MockTask, PrintTask
 
-__version__ = '2.11.0'
+__version__ = '2.12.0'

taskqueue/aws_queue_api.py

@@ -6,2 +6,3 @@ import json
 import boto3
+import botocore.exceptions
 import botocore.errorfactory
@@ -15,4 +16,16 @@
 
+import tenacity
+
 AWS_BATCH_SIZE = 10 # send_message_batch's max batch size is 10
 
+class ClientSideError(Exception):
+  pass
+
+retry = tenacity.retry(
+  reraise=True,
+  stop=tenacity.stop_after_attempt(4),
+  wait=tenacity.wait_random_exponential(0.5, 60.0),
+  retry=tenacity.retry_if_not_exception_type(ClientSideError),
+)
+
 class AWSTaskQueueAPI(object):
@@ -47,10 +60,14 @@ def __init__(self, qurl, region_name=AWS_DEFAULT_REGION, **kwargs):
     if self.qurl is None:
-      try:
-        self.qurl = self.sqs.get_queue_url(QueueName=qurl)["QueueUrl"]
-      except Exception:
-        print(qurl)
-        raise
+      self.qurl = self._get_qurl(qurl)
 
     self.batch_size = AWS_BATCH_SIZE
 
+  @retry
+  def _get_qurl(self, qurl):
+    try:
+      return self.sqs.get_queue_url(QueueName=qurl)["QueueUrl"]
+    except Exception as err:
+      print(f"Failed to fetch queue URL for: {qurl}")
+      raise    
+
   @property
@@ -77,2 +94,3 @@ def enqueued(self):
 
+  @retry
   def status(self):
@@ -85,2 +103,3 @@ resp = self.sqs.get_queue_attributes(
 
+  @retry
   def insert(self, tasks, delay_seconds=0):
@@ -102,6 +121,14 @@ tasks = toiter(tasks)
 
-      resp = self.sqs.send_message_batch(
-        QueueUrl=self.qurl,
-        Entries=entries,
-      )
+      try:
+        resp = self.sqs.send_message_batch(
+          QueueUrl=self.qurl,
+          Entries=entries,
+        )
+      except botocore.exceptions.ClientError as error:
+        http_code = error.response['ResponseMetadata']['HTTPStatusCode']
+        if 400 <= int(http_code) < 500:
+          raise ClientSideError(error)
+        else:
+          raise error
+      
       total += len(entries)
@@ -108,0 +135,0 @@

Alert delta unavailable

Currently unable to show alert delta for PyPI packages.

Improved metrics

Total package byte prevSize

161614

1.09%

Lines of code

2273

0.98%