karton-core - npm Package Compare versions

Comparing version 5.6.1 to 5.7.0

karton_core-5.7.0-nspkg.pth

+import sys, types, os;has_mfs = sys.version_info > (3, 5);p = os.path.join(sys._getframe(1).f_locals['sitedir'], *('karton',));importlib = has_mfs and __import__('importlib.util');has_mfs and __import__('importlib.machinery');m = has_mfs and sys.modules.setdefault('karton', importlib.util.module_from_spec(importlib.machinery.PathFinder.find_spec('karton', [os.path.dirname(p)])));m = m or sys.modules.setdefault('karton', types.ModuleType('karton'));mp = (m or []) and m.__dict__.setdefault('__path__',[]);(p not in mp) and mp.append(p)

karton/core/__version__.py

@@ -1,1 +0,1 @@
-__version__ = "5.6.1"
+__version__ = "5.7.0"

karton/core/backend.py

@@ -1043,5 +1043,7 @@ import dataclasses
             [
-                {"Key": uid, "VersionId": version_id}
-                if version_id != "null" or explicit_version_null
-                else {"Key": uid}
+                (
+                    {"Key": uid, "VersionId": version_id}
+                    if version_id != "null" or explicit_version_null
+                    else {"Key": uid}
+                )
                 for uid, versions in object_versions.items()
@@ -1074,3 +1076,5 @@ for version_id in versions
 
-    def log_identity_output(self, identity: str, headers: Dict[str, Any]) -> None:
+    def log_identity_output(
+        self, identity: str, headers: Dict[str, Any], task_tracking_ttl: int
+    ) -> None:
         """
@@ -1082,6 +1086,7 @@ Store the type of task outputted for given producer to
         :param headers: outputted headers
+        :param task_tracking_ttl: expire time (in seconds)
         """
 
         self.redis.sadd(f"{KARTON_OUTPUTS_NAMESPACE}:{identity}", json.dumps(headers))
-        self.redis.expire(f"{KARTON_OUTPUTS_NAMESPACE}:{identity}", 60 * 60 * 24 * 30)
+        self.redis.expire(f"{KARTON_OUTPUTS_NAMESPACE}:{identity}", task_tracking_ttl)
 
@@ -1088,0 +1093,0 @@ def get_outputs(self) -> List[KartonOutputs]:

karton/core/base.py

@@ -39,2 +39,5 @@ import abc
         self.config = config or Config()
+        self.enable_publish_log = self.config.getboolean(
+            "logging", "enable_publish", True
+        )
 
@@ -112,3 +115,3 @@ # If not passed via constructor - get it from class
 
-        if not self.debug:
+        if not self.debug and self.enable_publish_log:
             logger.addHandler(self._log_handler)
@@ -115,0 +118,0 @@

karton/core/config.py

@@ -114,8 +114,6 @@ import configparser
     @overload
-    def getint(self, section_name: str, option_name: str, fallback: int) -> int:
-        ...
+    def getint(self, section_name: str, option_name: str, fallback: int) -> int: ...
 
     @overload
-    def getint(self, section_name: str, option_name: str) -> Optional[int]:
-        ...
+    def getint(self, section_name: str, option_name: str) -> Optional[int]: ...
 
@@ -135,8 +133,8 @@ def getint(
     @overload
-    def getboolean(self, section_name: str, option_name: str, fallback: bool) -> bool:
-        ...
+    def getboolean(
+        self, section_name: str, option_name: str, fallback: bool
+    ) -> bool: ...
 
     @overload
-    def getboolean(self, section_name: str, option_name: str) -> Optional[bool]:
-        ...
+    def getboolean(self, section_name: str, option_name: str) -> Optional[bool]: ...
 
@@ -143,0 +141,0 @@ def getboolean(

karton/core/karton.py

 """
 Base library for karton subsystems.
 """
+
 import abc
@@ -5,0 +6,0 @@ import argparse

karton/core/resource.py

@@ -42,2 +42,3 @@ import contextlib
         sha256: Optional[str] = None,
+        fd: Optional[IO[bytes]] = None,
         _uid: Optional[str] = None,
@@ -70,2 +71,12 @@ _size: Optional[int] = None,
                 sha256 = sha256_hash.hexdigest()
+        elif fd is not None:
+            if calculate_hash:
+                # we need to calculate the whole hash and return pos as it was
+                sha256_hash = hashlib.sha256()
+                last_position = fd.tell()
+                fd.seek(0)
+                for byte_block in iter(lambda: fd.read(4096), b""):
+                    sha256_hash.update(byte_block)
+                sha256 = sha256_hash.hexdigest()
+                fd.seek(last_position)
         elif content:
@@ -195,2 +206,3 @@ if isinstance(content, str):
             sha256=sha256,
+            fd=fd,
             _uid=uid,
@@ -197,0 +209,0 @@ _flags=_flags,

karton/core/test.py

 """
 Test stubs for karton subsystem unit tests
 """
+
 import hashlib
@@ -8,3 +9,3 @@ import logging
 from collections import defaultdict
-from typing import Any, BinaryIO, Dict, List, Union, cast
+from typing import Any, BinaryIO, Dict, List, Optional, Union, cast
 from unittest import mock
@@ -70,3 +71,3 @@
         content: Union[bytes, BinaryIO],
-        length: int = None,
+        length: Optional[int] = None,
     ) -> None:
@@ -73,0 +74,0 @@ log.debug("Uploading object %s to bucket %s", object_uid, bucket)

karton/system/system.py

@@ -34,2 +34,3 @@ import argparse
     TASK_CRASHED_TIMEOUT = 3 * 24 * 3600
+    TASK_TRACKING_TTL = 30 * 24 * 3600
 
@@ -56,2 +57,8 @@ def __init__(self, config: Optional[Config]) -> None:
         )
+        self.enable_task_tracking = self.config.getboolean(
+            "system", "enable_task_tracking", True
+        )
+        self.task_tracking_ttl = self.config.getint(
+            "system", "task_tracking_ttl", self.TASK_TRACKING_TTL
+        )
 
@@ -70,2 +77,4 @@ self.last_gc_trigger = time.time()
             " enable_null_version_deletion:\t%s\n"
+            " enable_task_tracking:\t%s\n"
+            " task_tracking_ttl:\t%s\n"
             " crash_started_tasks_on_timeout:\t%s",
@@ -79,2 +88,4 @@ self.gc_interval,
             self.enable_null_version_deletion,
+            self.enable_task_tracking,
+            self.task_tracking_ttl,
             self.crash_started_tasks_on_timeout,
@@ -234,6 +245,9 @@ )
         self.log.info("[%s] Processing task %s", task.root_uid, task.task_uid)
-        # store the producer-task relationship in redis for task tracking
-        self.backend.log_identity_output(
-            task.headers.get("origin", "unknown"), task.headers
-        )
+        # if enabled, store the producer-task relationship in redis for task tracking
+        if self.enable_task_tracking:
+            self.backend.log_identity_output(
+                task.headers.get("origin", "unknown"),
+                task.headers,
+                self.task_tracking_ttl,
+            )
 
@@ -293,5 +307,6 @@ pipe = self.backend.make_pipeline()
         self.backend.register_tasks(tasks)
-        self.backend.produce_logs(
-            operation_bodies, logger_name=KARTON_OPERATIONS_QUEUE, level="INFO"
-        )
+        if self.enable_publish_log:
+            self.backend.produce_logs(
+                operation_bodies, logger_name=KARTON_OPERATIONS_QUEUE, level="INFO"
+            )
 
@@ -298,0 +313,0 @@ def process_routing(self) -> None:

METADATA

 Metadata-Version: 2.1
 Name: karton-core
-Version: 5.6.1
+Version: 5.7.0
 Summary: Distributed malware analysis orchestration framework
@@ -5,0 +5,0 @@ Home-page: https://github.com/CERT-Polska/karton

RECORD

@@ -1,10 +0,10 @@
-karton_core-5.6.1-nspkg.pth,sha256=vHa-jm6pBTeInFrmnsHMg9AOeD88czzQy-6QCFbpRcM,539
+karton_core-5.7.0-nspkg.pth,sha256=vHa-jm6pBTeInFrmnsHMg9AOeD88czzQy-6QCFbpRcM,539
 karton/core/__init__.py,sha256=QuT0BWZyp799eY90tK3H1OD2hwuusqMJq8vQwpB3kG4,337
-karton/core/__version__.py,sha256=-q9tSF5ofTJum4PMjvbhaE1xmTXehc_9rxMGcmfodcw,22
-karton/core/backend.py,sha256=g0BSQBsFAksRd_VY5QDjBJ8yIIyzAmwxy-kfJgAZ_lo,38628
-karton/core/base.py,sha256=C6Lco3E0XCsxvEjeVOLR9fxh_IWJ1vjC9BqUYsQyewE,8083
-karton/core/config.py,sha256=7oKchitq6pWzPuXRfjBXqVT_BgGIz2p-CDo1RGaNJQg,8118
+karton/core/__version__.py,sha256=QmHMXVnw5DVPfWzvN7FS1tOhDAesdxpM_aVOh9CMuSk,22
+karton/core/backend.py,sha256=_IOjN9pWdSBsDnTMYvg-Fpm6Ag-uf2Jb9LWmrtZqVAU,38773
+karton/core/base.py,sha256=lqVJvCHRMzvIOpS8SaWlOaSSJBEVkNQe0oClZC_GQYM,8225
+karton/core/config.py,sha256=M3dB0XgnUO5VzUcGyQa7FyKzmdgmDml1MrzG6CxEuvE,8100
 karton/core/exceptions.py,sha256=8i9WVzi4PinNlX10Cb-lQQC35Hl-JB5R_UKXa9AUKoQ,153
 karton/core/inspect.py,sha256=aIJQEOEkD5q2xLlV8nhxY5qL5zqcnprP-2DdP6ecKlE,6150
-karton/core/karton.py,sha256=Fi3wNqMGiKvHN2BECsqsvfxkiyuwPdlC21jpqQdkeak,15434
+karton/core/karton.py,sha256=l3joJWw8m23wlOErkcQmNFYhLFA5x2la6L0WopxJ7mk,15435
 karton/core/logger.py,sha256=J3XAyG88U0cwYC9zR6E3QD1uJenrQh7zS9-HgxhqeAs,2040
@@ -14,15 +14,15 @@ karton/core/main.py,sha256=ir1-dhn3vbwfh2YHiM6ZYfRBbjwLvJSz0d8tuK1mb_4,8310
 karton/core/query.py,sha256=sf24DweVlXfJuBbBD_ns2LXhOV-IBwuPG3jBfTJu77s,12063
-karton/core/resource.py,sha256=tA3y_38H9HVKIrCeAU70zHUkQUv0BuCQWMC470JLxxc,20321
+karton/core/resource.py,sha256=9kWXpMBRfudH0_whJfSSI27K3Gwv2u93CVa7p68Q5UM,20842
 karton/core/task.py,sha256=gW1szMi5PN2Y06X-Ryo7cmEVluZv1r7W5tvmwIJiD94,18808
-karton/core/test.py,sha256=tms-YM7sUKQDHN0vm2_W7DIvHnO_ld_VPsWHnsbKSfk,9102
+karton/core/test.py,sha256=cj6W4gNt0BpRjsYiiBt0hPE8dmRfUeIc8sSVkxB50cU,9123
 karton/core/utils.py,sha256=sEVqGdVPyYswWuVn8wYXBQmln8Az826N_2HgC__pmW8,4090
 karton/system/__init__.py,sha256=JF51OqRU_Y4c0unOulvmv1KzSHSq4ZpXU8ZsH4nefRM,63
 karton/system/__main__.py,sha256=QJkwIlSwaPRdzwKlNmCAL41HtDAa73db9MZKWmOfxGM,56
-karton/system/system.py,sha256=cFE4hCS0LWnwdCiIjU0ym8dHujE5ORi4REJR_y5b2gA,16671
-karton_core-5.6.1.dist-info/LICENSE,sha256=o8h7hYhn7BJC_-DmrfqWwLjaR_Gbe0TZOOQJuN2ca3I,1519
-karton_core-5.6.1.dist-info/METADATA,sha256=AJoa9O_0SOYI3IuVHXhwB6lXoUSs7S4nU6QM8_xHxVI,6818
-karton_core-5.6.1.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
-karton_core-5.6.1.dist-info/entry_points.txt,sha256=OgLlsXy61GP6-Yob3oXqeJ2hlRU6LBLj33fr0NufKz0,98
-karton_core-5.6.1.dist-info/namespace_packages.txt,sha256=X8SslCPsqXDCnGZqrYYolzT3xPzJMq1r-ZQSc0jfAEA,7
-karton_core-5.6.1.dist-info/top_level.txt,sha256=X8SslCPsqXDCnGZqrYYolzT3xPzJMq1r-ZQSc0jfAEA,7
-karton_core-5.6.1.dist-info/RECORD,,
+karton/system/system.py,sha256=d_5hhLTthJdr_4gZEGQ6Y-kHvxeBqyQxjjx_wRs3xMA,17285
+karton_core-5.7.0.dist-info/LICENSE,sha256=o8h7hYhn7BJC_-DmrfqWwLjaR_Gbe0TZOOQJuN2ca3I,1519
+karton_core-5.7.0.dist-info/METADATA,sha256=MrmtycTaYsNB8v0LRyuLIHL2bV17n1Lt6e-ak4RfrH8,6818
+karton_core-5.7.0.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
+karton_core-5.7.0.dist-info/entry_points.txt,sha256=OgLlsXy61GP6-Yob3oXqeJ2hlRU6LBLj33fr0NufKz0,98
+karton_core-5.7.0.dist-info/namespace_packages.txt,sha256=X8SslCPsqXDCnGZqrYYolzT3xPzJMq1r-ZQSc0jfAEA,7
+karton_core-5.7.0.dist-info/top_level.txt,sha256=X8SslCPsqXDCnGZqrYYolzT3xPzJMq1r-ZQSc0jfAEA,7
+karton_core-5.7.0.dist-info/RECORD,,

karton_core-5.6.1-nspkg.pth

-import sys, types, os;has_mfs = sys.version_info > (3, 5);p = os.path.join(sys._getframe(1).f_locals['sitedir'], *('karton',));importlib = has_mfs and __import__('importlib.util');has_mfs and __import__('importlib.machinery');m = has_mfs and sys.modules.setdefault('karton', importlib.util.module_from_spec(importlib.machinery.PathFinder.find_spec('karton', [os.path.dirname(p)])));m = m or sys.modules.setdefault('karton', types.ModuleType('karton'));mp = (m or []) and m.__dict__.setdefault('__path__',[]);(p not in mp) and mp.append(p)

Alert delta unavailable

Currently unable to show alert delta for PyPI packages.

No metric changes