quilt - npm Package Compare versions

Comparing version 2.9.11 to 2.9.12

PKG-INFO

 Metadata-Version: 1.1
 Name: quilt
-Version: 2.9.11
+Version: 2.9.12
 Summary: Quilt is a data package manager
@@ -9,3 +9,3 @@ Home-page: https://github.com/quiltdata/quilt
 License: LICENSE
-Download-URL: https://github.com/quiltdata/quilt/releases/tag/2.9.11
+Download-URL: https://github.com/quiltdata/quilt/releases/tag/2.9.12
 Description-Content-Type: UNKNOWN
@@ -12,0 +12,0 @@ Description:

quilt.egg-info/PKG-INFO

 Metadata-Version: 1.1
 Name: quilt
-Version: 2.9.11
+Version: 2.9.12
 Summary: Quilt is a data package manager
@@ -9,3 +9,3 @@ Home-page: https://github.com/quiltdata/quilt
 License: LICENSE
-Download-URL: https://github.com/quiltdata/quilt/releases/tag/2.9.11
+Download-URL: https://github.com/quiltdata/quilt/releases/tag/2.9.12
 Description-Content-Type: UNKNOWN
@@ -12,0 +12,0 @@ Description:

quilt/test/test_build.py

@@ -215,3 +215,3 @@ """
         with open(buildfilepath) as fd:
-            docs = yaml.load_all(fd)
+            docs = yaml.safe_load_all(fd)
             data = next(docs, None)
@@ -495,1 +495,19 @@
             command.build('test/foo', arr)
+
+class TestOutput(QuiltTestCase):
+    # This is kindof a quirky workaround. 'capsys' and 'capfd' fixtures don't work
+    # with unittest.TestCase objects, but they do work with other fixtures -- including
+    # those defined in this scope.
+    @pytest.fixture(autouse=True)
+    def capfd(self, capfd):
+        self.capfd = capfd
+
+    def test_read_yaml_exec_flaw(self):
+        # We don't execute anything remote, but someone could give a bad build.yml..
+        testdir = pathlib.Path(__file__).parent
+
+        with pytest.raises(yaml.constructor.ConstructorError):
+            command.build('test/exec_flaw', str(testdir / 'arbitrary_execution.yml'), build_file=True)
+        out, err = self.capfd.readouterr()
+        assert not "arbitrary code execution" in out
+        assert not "arbitrary code execution" in err

quilt/test/test_checks.py

@@ -18,3 +18,3 @@ """
     with open(filepath) as fd:
-        return next(yaml.load_all(fd), None)
+        return next(yaml.safe_load_all(fd), None)
 
@@ -52,3 +52,3 @@ class ChecksTest(QuiltTestCase):
             self.build_success(check, nodename=nodename)
-        
+
     def test_parse_checks_file(self):
@@ -65,3 +65,3 @@ assert str(self.checks_contents['negative']) == 'False'
         self.build_fail('inline_only', "Unknown check.+inline_only")
-        
+
     def test_inline_only(self):
@@ -71,3 +71,3 @@ self.checks_contents = self.checks_data = None
         self.build_fail('hasrecs', "Unknown check.+hasrecs")
-        
+
     def test_simple_checks(self):
@@ -94,3 +94,3 @@ self.build_success('simple')
         self.build_success('hasrecs')
-        
+
     def test_many_errors(self):
@@ -97,0 +97,0 @@ # TODO: capture details by line number

quilt/test/test_store.py

@@ -34,1 +34,11 @@ """
             assert fd.read() == '1.3'
+
+    def test_buggy_parquet(self):
+        mydir = os.path.dirname(__file__)
+        shutil.copytree(os.path.join(mydir, 'store_buggy_parquet'), self._store_dir)
+
+        store, pkg = PackageStore.find_package(None, 'test', 'bug')
+        obj_hashes = pkg.children['bug'].hashes
+
+        # Make sure this doesn't crash.
+        store.load_dataframe(obj_hashes)

quilt/tools/build.py

@@ -620,3 +620,3 @@ """
     try:
-        res = yaml.load(data)
+        res = yaml.safe_load(data)
         if not filename.endswith(DEFAULT_QUILT_YML):
@@ -623,0 +623,0 @@ if 'contents' not in res.keys():

quilt/tools/command.py

@@ -870,3 +870,3 @@ # -*- coding: utf-8 -*-
     else:
-        yaml_data = yaml.load(requirements_str)
+        yaml_data = yaml.safe_load(requirements_str)
     for pkginfo in yaml_data['packages']:
@@ -873,0 +873,0 @@ info = parse_package_extended(pkginfo)

quilt/tools/core.py

@@ -93,3 +93,5 @@ # Copyright (c) 2017 Quilt Data, Inc. All rights reserved.
 
-        assert PackageFormat(format) == PackageFormat.PARQUET
+        # This AssertionError needed with this message to catch later and present clear user information.
+        if not PackageFormat(format) == PackageFormat.PARQUET:
+            raise AssertionError("Bad package format '{}', this package may be outdated.".format(format))
 
@@ -96,0 +98,0 @@ assert isinstance(hashes, list)

quilt/tools/data_transfer.py

@@ -20,3 +20,3 @@ """
 from .hashing import digest_file
-from .util import FileWithReadProgress, get_free_space
+from .util import FileWithReadProgress, get_free_space, QuiltException
 
@@ -88,3 +88,6 @@
                         obj_hash, url = obj_queue.pop()
-                        original_size = obj_sizes[obj_hash] or 0  # If the size is unknown, just treat it as 0.
+                        try:
+                            original_size = obj_sizes[obj_hash]
+                        except KeyError:
+                            raise QuiltException("Malformed package: No size for object " + obj_hash)
 
@@ -100,2 +103,29 @@ success = False
 
+                                # For zero-byte downloads, we don't need to resume, and range download
+                                # must be at least 1 anyways.
+                                if original_size == 0:
+                                    response = s3_session.get(
+                                        url,
+                                        timeout=(S3_CONNECT_TIMEOUT, S3_READ_TIMEOUT)
+                                    )
+                                    if not response.ok:
+                                        message = (
+                                            "Download failed for {obj_hash}:\n"
+                                            "URL: {response.request.url}\n"
+                                            "Status code: {response.status_code}\n"
+                                            "Response: {response.text!r}\n"
+                                        ).format(**locals())  # Splat **kwargs operators ftw
+                                        with lock:
+                                            tqdm.write(message)
+                                        break
+                                    if len(response.content):
+                                        message = "Expected a zero-byte file, but received content from: "
+                                        with lock:
+                                            tqdm.write(message + response.url)
+                                        break
+                                    encoding = response.headers.get('Content-Encoding', None)
+                                    # Nothing to write to the filesystem, already created by opening.
+                                    success = True
+                                    break
+
                                 # Use the Range header to resume downloads.
@@ -102,0 +132,0 @@ # Weird corner case: if the file is already completely downloaded, we will

quilt/tools/store.py

@@ -206,3 +206,3 @@ """
 
-        assert pkghash is not None  
+        assert pkghash is not None
         contents_path = os.path.join(path, self.CONTENTS_DIR, pkghash)
@@ -213,3 +213,15 @@ if not os.path.isfile(contents_path):
         with open(contents_path, 'r') as contents_file:
-            return json.load(contents_file, object_hook=decode_node)
+            try:
+                return json.load(contents_file, object_hook=decode_node)
+            except AssertionError as err:
+                if str(err).startswith("Bad package format"):
+                    name = "{}{}/{}, {}".format(
+                        team + ':' if team else '',
+                        user,
+                        package,
+                        pkghash
+                        )
+                    raise StoreException("Error in {}: {}".format(name, str(err)))
+                else:
+                    raise
 
@@ -232,3 +244,3 @@ def install_package(self, team, user, package, contents):
             pass
-    
+
     def create_package_node(self, team, user, package, dry_run=False):
@@ -381,4 +393,7 @@ """
         dataset = ParquetDataset(objfiles)
-        table = dataset.read(nthreads=4)
         try:
+            table = dataset.read(use_threads=True)  # pyarrow == 0.11
+        except TypeError:
+            table = dataset.read(nthreads=4)  # pyarrow < 0.11
+        try:
             dataframe = table.to_pandas()
@@ -548,3 +563,3 @@ except Exception:
             os.mkdir(os.path.join(pkg_path, self.VERSIONS_DIR))
-            
+
         dest = os.path.join(pkg_path, self.CONTENTS_DIR, instance_hash)
@@ -581,5 +596,5 @@ with open(dest, 'w') as contents_file:
         assert user_meta_hash is None or isinstance(user_meta_hash, str)
-        
+
         contents = pkgroot
-        
+
         if not node_path:
@@ -612,4 +627,4 @@ # Allow setting metadata on the root node, but that's it.
         ptr.children[node_path[-1]] = node
-        
 
+
 ########################################
@@ -663,2 +678,2 @@ # Methods ported from save_<xyz>
                 raise PackageException("Attempting to overwrite root node of a non-empty package.")
-            root.children = pkgnode.children.copy()   
+            root.children = pkgnode.children.copy()

setup.py

@@ -19,3 +19,3 @@ from setuptools import setup, find_packages
     name="quilt",
-    version="2.9.11",
+    version="2.9.12",
     packages=find_packages(),
@@ -37,3 +37,3 @@ description='Quilt is a data package manager',
     url='https://github.com/quiltdata/quilt',
-    download_url='https://github.com/quiltdata/quilt/releases/tag/2.9.11',
+    download_url='https://github.com/quiltdata/quilt/releases/tag/2.9.12',
     keywords='quilt quiltdata shareable data dataframe package platform pandas',
@@ -40,0 +40,0 @@ install_requires=[

Alert delta unavailable

Currently unable to show alert delta for PyPI packages.

Improved metrics

Total package byte prevSize

394858

0.98%

Lines of code

8632

0.79%