amira - npm Package Compare versions

Comparing version 1.1.5 to 1.2.0

amira/data_processor.py

+# -*- coding: utf-8 -*-
+from __future__ import absolute_import
+from __future__ import unicode_literals
+
+import logging
+import os
+import tarfile
+
+try:
+    from cStringIO import StringIO as ByteBuffer
+    from cStringIO import StringIO as StringBuffer
+except ImportError:
+    from io import BytesIO as ByteBuffer
+    from io import StringIO as StringBuffer
+
+from osxcollector.output_filters.analyze import AnalyzeFilter
+from osxcollector.output_filters.base_filters import output_filter
+
+from amira.results_uploader import FileMetaInfo
+
+
+class DataProcessor(object):
+
+    def __init__(self):
+        # List to store processing outputs
+        self._results = []
+
+    def process_input(self, tardata):
+        """Process input TAR file
+
+        :param tardata: TAR byte stream
+        :return: processed data file stream
+        """
+        raise NotImplementedError()
+
+    def perform_analysis(self, input_stream, data_feeds=None):
+        """Perform analysis of forensic input.
+        Analysis results should be handled as internal object state
+
+        :param input_stream: forensic data
+        :param data_feeds: additional data feeds which may be required in the analysis
+        """
+        raise NotImplementedError()
+
+    def upload_results(self, file_basename, result_uploaders):
+        """Upload forensic results.
+        These must be stored as FileMetaInfo objects in the `_results` list attribute
+
+        :param file_basename: Basename used to generate output filenames (prepended)
+        :param result_uploaders: List of Uploader objects to invoke
+        """
+        results = [
+            FileMetaInfo(file_basename + res.name, res.content, res.content_type) for res in self._results
+            if isinstance(res, FileMetaInfo) and DataProcessor.get_buffer_size(res.content) > 0
+        ]
+        if results:
+            for res_uploader in result_uploaders:
+                res_uploader.upload_results(results)
+        else:
+            logging.warning('No results to upload for {}'.format(file_basename))
+
+    @staticmethod
+    def get_buffer_size(data_buffer):
+        """Get byte size of file-like object
+
+        :param data_buffer: file-like object
+        :return: total size in bytes
+        """
+        data_buffer.seek(0, os.SEEK_END)
+        size = data_buffer.tell()
+        data_buffer.seek(0)
+        return size
+
+
+class OSXCollectorDataProcessor(DataProcessor):
+
+    def process_input(self, tardata):
+        """Extracts JSON file containing the OSXCollector output from
+        tar.gz archive. It will look in the archive contents for the
+        file with the extension ".json". If no file with this extension
+        is found in the archive or more than one JSON file is found, it
+        will raise `OSXCollectorOutputExtractionError`.
+
+        :param tardata: Input TAR archive data
+        """
+        self._results = [FileMetaInfo('.tar.gz', ByteBuffer(tardata), 'application/gzip')]
+        # create a file-like object based on the S3 object contents as string
+        fileobj = ByteBuffer(tardata)
+        tar = tarfile.open(mode='r:gz', fileobj=fileobj)
+        json_tarinfo = [t for t in tar if t.name.endswith('.json')]
+
+        if len(json_tarinfo) != 1:
+            raise OSXCollectorOutputExtractionError(
+                'Expected 1 JSON file inside the OSXCollector output archive, '
+                'but found {0} instead.'.format(len(json_tarinfo)),
+            )
+
+        tarinfo = json_tarinfo[0]
+        logging.info('Extracted OSXCollector output JSON file {0}'.format(tarinfo.name))
+        return tar.extractfile(tarinfo)
+
+    def perform_analysis(self, input_stream, data_feeds=None):
+        """Runs Analyze Filter on the OSXCollector output retrieved
+        from an S3 bucket.
+
+        :param input_stream: Input data stream on which filters should be ran
+        :param data_feeds: black/whitelist data feeds
+        """
+        analysis_output = StringBuffer()
+        text_analysis_summary = ByteBuffer()
+        html_analysis_summary = ByteBuffer()
+
+        analyze_filter = AnalyzeFilter(
+            monochrome=True,
+            text_output_file=text_analysis_summary,
+            html_output_file=html_analysis_summary,
+            data_feeds=data_feeds or {},
+        )
+
+        output_filter._run_filter(
+            analyze_filter,
+            input_stream=input_stream,
+            output_stream=analysis_output,
+        )
+
+        # rewind the output files
+        analysis_output.seek(0)
+        text_analysis_summary.seek(0)
+        html_analysis_summary.seek(0)
+
+        self._results += [
+            FileMetaInfo('_analysis.json', analysis_output, 'application/json'),
+            FileMetaInfo('_summary.txt', text_analysis_summary, 'text/plain'),
+            FileMetaInfo('_summary.html', html_analysis_summary, 'text/html; charset=UTF-8'),
+        ]
+
+
+class OSXCollectorOutputExtractionError(Exception):
+    """Raised when an unexpected number of JSON files is found in the
+    OSXCollector output archive.
+    """
+    pass

amira.egg-info/PKG-INFO

 Metadata-Version: 2.1
 Name: amira
-Version: 1.1.5
+Version: 1.2.0
 Summary: Automated Malware Incident Response and Analysis
@@ -5,0 +5,0 @@ Home-page: https://github.com/Yelp/amira

amira.egg-info/SOURCES.txt

@@ -7,2 +7,3 @@ LICENSE.md
 amira/amira.py
+amira/data_processor.py
 amira/results_uploader.py
@@ -9,0 +10,0 @@ amira/s3.py

amira/__init__.py

 # -*- coding: utf-8 -*-
 from __future__ import absolute_import
 from __future__ import unicode_literals
-__version__ = '1.1.5'
+
+__version__ = '1.2.0'

amira/amira.py

@@ -6,16 +6,4 @@ # -*- coding: utf-8 -*-
 import logging
-import os
-import tarfile
 
-try:
-    from cStringIO import StringIO as ByteBuffer
-    from cStringIO import StringIO as StringBuffer
-except ImportError:
-    from io import BytesIO as ByteBuffer
-    from io import StringIO as StringBuffer
-
-from osxcollector.output_filters.analyze import AnalyzeFilter
-from osxcollector.output_filters.base_filters import output_filter
-
-from amira.results_uploader import FileMetaInfo
+from amira.data_processor import OSXCollectorDataProcessor
 from amira.s3 import S3Handler
@@ -25,3 +13,3 @@ from amira.sqs import SqsHandler
 
-class AMIRA():
+class AMIRA(object):
     """Runs the automated analysis based on the new elements in an S3
@@ -56,2 +44,3 @@ bucket:
         self._data_feeds = {}
+        self._data_processor = OSXCollectorDataProcessor()
 
@@ -72,6 +61,14 @@ def register_results_uploader(self, results_uploader):
         :param generator: Generator function providing the data
-        :return:
         """
         self._data_feeds[feed_name] = generator
 
+    def register_data_processor(self, processor):
+        """Registers DataProcessor object to process and analyze input data from S3.
+        If no processor is registered Amira will fall back using the default
+        OSXCollector result processor.
+
+        :param processor: DataProcessor object instance
+        """
+        self._data_processor = processor
+
     def run(self):
@@ -100,10 +97,10 @@ """Fetches the OSXCollector output from an S3 bucket based on
         """
-        # fetch the OSXCollector output from the S3 bucket
-        self._osxcollector_output = self._s3_handler.get_contents_as_string(
+        # fetch forensic data from the S3 bucket
+        forensic_output = self._s3_handler.get_contents_as_string(
             created_object.bucket_name, created_object.key_name,
         )
-        self._extract_osxcollector_output_json_file()
+        processed_input = self._data_processor.process_input(forensic_output)
 
         try:
-            self._run_analyze_filter()
+            self._data_processor.perform_analysis(processed_input)
         except Exception as exc:
@@ -119,3 +116,5 @@ # Log the exception and do not try any recovery.
         try:
-            self._upload_analysis_results(created_object.key_name)
+            self._data_processor.upload_results(
+                created_object.key_name[:-7], self._results_uploader,
+            )
         except Exception:
@@ -126,103 +125,1 @@ logging.exception(
             )
-
-    def _extract_osxcollector_output_json_file(self):
-        """Extracts JSON file containing the OSXCollector output from
-        tar.gz archive. It will look in the archive contents for the
-        file with the extension ".json". If no file with this extension
-        is found in the archive or more than one JSON file is found, it
-        will raise `OSXCollectorOutputExtractionError`.
-        """
-        # create a file-like object based on the S3 object contents as string
-        fileobj = ByteBuffer(self._osxcollector_output)
-        tar = tarfile.open(mode='r:gz', fileobj=fileobj)
-        json_tarinfo = [t for t in tar if t.name.endswith('.json')]
-
-        if 1 != len(json_tarinfo):
-            raise OSXCollectorOutputExtractionError(
-                'Expected 1 JSON file inside the OSXCollector output archive, '
-                'but found {0} instead.'.format(len(json_tarinfo)),
-            )
-
-        tarinfo = json_tarinfo[0]
-        self._osxcollector_output_json_file = tar.extractfile(tarinfo)
-        logging.info(
-            'Extracted OSXCollector output JSON file {0}'.format(tarinfo.name),
-        )
-
-    def _run_analyze_filter(self):
-        """Runs Analyze Filter on the OSXCollector output retrieved
-        from an S3 bucket.
-        """
-        self._analysis_output = StringBuffer()
-        self._text_analysis_summary = ByteBuffer()
-        self._html_analysis_summary = ByteBuffer()
-
-        analyze_filter = AnalyzeFilter(
-            monochrome=True,
-            text_output_file=self._text_analysis_summary,
-            html_output_file=self._html_analysis_summary,
-            data_feeds=self._data_feeds,
-        )
-
-        output_filter._run_filter(
-            analyze_filter,
-            input_stream=self._osxcollector_output_json_file,
-            output_stream=self._analysis_output,
-        )
-
-        # rewind the output files
-        self._analysis_output.seek(0)
-        self._text_analysis_summary.seek(0)
-        self._html_analysis_summary.seek(0)
-
-    @staticmethod
-    def _check_buffer_size(buffer):
-        buffer.seek(0, os.SEEK_END)
-        size = buffer.tell()
-        buffer.seek(0)
-        return size
-
-    def _upload_analysis_results(self, osxcollector_output_filename):
-        # drop the file extension (".tar.gz")
-        filename_without_extension = osxcollector_output_filename[:-7]
-
-        analysis_output_filename = '{0}_analysis.json'.format(
-            filename_without_extension,
-        )
-        text_analysis_summary_filename = '{0}_summary.txt'.format(
-            filename_without_extension,
-        )
-        html_analysis_summary_filename = '{0}_summary.html'.format(
-            filename_without_extension,
-        )
-
-        results = [
-            FileMetaInfo(
-                osxcollector_output_filename,
-                ByteBuffer(self._osxcollector_output), 'application/gzip',
-            ),
-            FileMetaInfo(
-                analysis_output_filename, self._analysis_output,
-                'application/json',
-            ),
-            FileMetaInfo(
-                text_analysis_summary_filename, self._text_analysis_summary,
-                'text/plain',
-            ),
-            FileMetaInfo(
-                html_analysis_summary_filename, self._html_analysis_summary,
-                'text/html; charset=UTF-8',
-            ),
-        ]
-        results = [res for res in results if AMIRA._check_buffer_size(res.content) > 0]
-
-        for results_uploader in self._results_uploader:
-            results_uploader.upload_results(results)
-
-
-class OSXCollectorOutputExtractionError(Exception):
-    """Raised when an unexpected number of JSON files is found in the
-    OSXCollector output archive.
-    """
-
-    pass

amira/results_uploader.py

@@ -11,3 +11,3 @@ # -*- coding: utf-8 -*-
 
-class ResultsUploader():
+class ResultsUploader(object):
 
@@ -14,0 +14,0 @@ """Parent class for the AMIRA results uploaders. Results uploaders

amira/s3.py

@@ -13,3 +13,3 @@ # -*- coding: utf-8 -*-
 
-class S3Handler():
+class S3Handler(object):
     """Handles the operations with S3, like retrieving the key
@@ -16,0 +16,0 @@ (object) contents from a bucket and creating a new key

amira/sqs.py

@@ -21,3 +21,3 @@ # -*- coding: utf-8 -*-
 
-class SqsHandler():
+class SqsHandler(object):
     """Retrieves the S3 event notifications about the objects created
@@ -24,0 +24,0 @@ in the bucket for which the notifications were configured.

PKG-INFO

 Metadata-Version: 2.1
 Name: amira
-Version: 1.1.5
+Version: 1.2.0
 Summary: Automated Malware Incident Response and Analysis
@@ -5,0 +5,0 @@ Home-page: https://github.com/Yelp/amira

Alert delta unavailable

Currently unable to show alert delta for PyPI packages.

Improved metrics

Total package byte prevSize

38450

3.99%

Number of package files

17

6.25%

Lines of code

456

5.8%