Socket
Book a DemoSign in
Socket

datary

Package Overview
Dependencies
Maintainers
1
Versions
62
Alerts
File Explorer

Advanced tools

Socket logo

Install Socket

Detect and block malicious and high-risk dependencies

Install

datary - pypi Package Compare versions

Comparing version
0.1.28
to
0.1.29
+5
datary/workdirs/__init__.py
# -*- coding: utf-8 -*-
"""
Datary workdirs Module
"""
from .workdirs import DataryWorkdirs
# -*- coding: utf-8 -*-
"""
Datary python sdk workdirs test file
"""
import mock
from datary.test.test_datary import DataryTestCase
from datary.test.mock_requests import MockRequestResponse
class DataryWorkdirsTestCase(DataryTestCase):
"""
Datary workdirs Test Case
"""
@mock.patch('datary.Datary.request')
def test_get_commit_filetree(self, mock_request):
"""
Test get_commit_filetree
"""
mock_request.return_value = MockRequestResponse(
"", json=self.wdir_json.get('workdir'))
workdir = self.datary.get_commit_filetree(
self.repo_uuid, self.commit_sha1)
self.assertEqual(mock_request.call_count, 1)
assert isinstance(workdir, dict)
@mock.patch('datary.Datary.request')
def test_get_wdir_filetree(self, mock_request):
"""
Test get_wdir_filetree
"""
mock_request.return_value = MockRequestResponse(
"", json=self.wdir_json.get('changes'))
changes = self.datary.get_wdir_filetree(self.repo_uuid)
self.assertEqual(mock_request.call_count, 1)
assert isinstance(changes, dict)
@mock.patch('datary.Datary.request')
@mock.patch('datary.repos.DataryRepos.get_describerepo')
def test_get_wdir_changes(self, mock_describerepo, mock_request):
"""
Test get_wdir_changes
"""
mock_request.return_value = MockRequestResponse(
"", json=self.wdir_json.get('workdir'))
workdir = self.datary.get_wdir_changes(self.wdir_uuid)
self.assertEqual(mock_request.call_count, 1)
assert isinstance(workdir, dict)
mock_describerepo.return_value = self.json_repo
mock_request.return_value = MockRequestResponse(
"", json=self.wdir_json.get('workdir'))
workdir = self.datary.get_wdir_changes(repo_uuid=self.repo_uuid)
self.assertEqual(mock_request.call_count, 2)
assert isinstance(workdir, dict)
def test_format_wdir_change(self):
"""
Test format_wdir_change
"""
format_filetree = self.datary.format_wdir_changes
treeformated_changes = format_filetree(self.changes.values())
self.assertEqual(len(treeformated_changes.keys()), 3)
self.assertEqual(treeformated_changes.get('a'), 'inode2_changes')
self.assertEqual(treeformated_changes.get('b'),
{'bb': 'inode1_changes'})
self.assertEqual(treeformated_changes.get('d'), 'inode3_changes')
# -*- coding: utf-8 -*-
"""
Datary sdk workdirs File
"""
import os
from urllib.parse import urljoin
from datary.repos import DataryRepos
from scrapbag import force_list, add_element
import structlog
logger = structlog.getLogger(__name__)
class DataryWorkdirs(DataryRepos):
"""
Datary workdirs module class
"""
def get_commit_filetree(self, repo_uuid, commit_sha1):
"""
============== ============= ====================================
Parameter Type Description
============== ============= ====================================
repo_uuid int repository id
commit_sha1 str workdir sha1
============== ============= ====================================
Returns:
workdir of all commits done in a repo.
"""
url = urljoin(self.URL_BASE,
"commits/{}/workdir".format(commit_sha1))
params = {'namespace': repo_uuid}
response = self.request(
url, 'GET', **{'headers': self.headers, 'params': params})
return response.json() if response else {}
def get_wdir_filetree(self, wdir_uuid):
"""
============== ============= ====================================
Parameter Type Description
============== ============= ====================================
wdir_uuid str working directory id
============== ============= ====================================
Returns:
workdir of a repo workdir.
"""
url = urljoin(self.URL_BASE,
"workdirs/{}/workdir".format(wdir_uuid))
response = self.request(url, 'GET', **{'headers': self.headers})
return response.json() if response else {}
def get_wdir_changes(self, wdir_uuid=None, **kwargs):
"""
================ ============= ====================================
Parameter Type Description
================ ============= ====================================
wdir_uuid str working directory id
================ ============= ====================================
Returns:
(dict) changes in workdir.
"""
# try to take wdir_uuid with kwargs
if not wdir_uuid:
wdir_uuid = self.get_describerepo(
**kwargs).get('workdir', {}).get('uuid')
url = urljoin(self.URL_BASE,
"workdirs/{}/changes".format(wdir_uuid))
response = self.request(url, 'GET', **{'headers': self.headers})
return response.json() if response else {}
def format_wdir_changes(self, wdir_changes_tree):
"""
================== ============= ==================================
Parameter Type Description
================== ============= ==================================
wdir_changes_tree list working changes tree
================== ============= ==================================
Returns:
(dict) changes in workdir formatting as workdir format.
"""
result = {}
for sublist in list(wdir_changes_tree):
for item in force_list(sublist):
add_element(
result,
os.path.join(item.get('dirname', ''),
item.get('basename', '')),
item.get('inode', 'unkwown_dataset_uuid')
)
return result
+1
-1
Metadata-Version: 1.1
Name: datary
Version: 0.1.28
Version: 0.1.29
Summary: Datary Python sdk lib

@@ -5,0 +5,0 @@ Home-page: https://github.com/Datary/python-sdk

@@ -29,6 +29,2 @@ MANIFEST.in

datary/datasets/test/test_datasets.py
datary/filetrees/__init__.py
datary/filetrees/filetrees.py
datary/filetrees/test/__init__.py
datary/filetrees/test/test_filetrees.py
datary/members/__init__.py

@@ -61,2 +57,6 @@ datary/members/members.py

datary/test/mock_requests.py
datary/test/test_datary.py
datary/test/test_datary.py
datary/workdirs/__init__.py
datary/workdirs/workdirs.py
datary/workdirs/test/__init__.py
datary/workdirs/test/test_workdirs.py

@@ -39,3 +39,3 @@ # -*- coding: utf-8 -*-

- DataryDatasets
- DataryFiletrees
- DataryWorkdirs
- DataryMembers

@@ -42,0 +42,0 @@ - DataryAddOperation

@@ -56,3 +56,3 @@

- No sha1 in repo.
- No filetree in repo.
- No workdir in repo.
- Fail retrieving last commit.

@@ -73,3 +73,3 @@

# retrieve last filetree commited
# retrieve last workdir commited
ftree = self.get_last_commit_filetree(repo)

@@ -76,0 +76,0 @@

@@ -32,3 +32,3 @@ # -*- coding: utf-8 -*-

@mock.patch('datary.repos.DataryRepos.get_describerepo')
@mock.patch('datary.filetrees.DataryFiletrees.get_commit_filetree')
@mock.patch('datary.workdirs.DataryWorkdirs.get_commit_filetree')
@mock.patch('datary.datasets.DataryDatasets.get_metadata')

@@ -46,3 +46,3 @@ def test_recollect_last_commit(self, mock_metadata, mock_filetree,

mock_filetree.return_value = self.filetree
mock_filetree.return_value = self.workdir
mock_get_describerepo.return_value = self.json_repo

@@ -49,0 +49,0 @@ mock_metadata.return_value.json.return_value = \

@@ -8,3 +8,3 @@ # -*- coding: utf-8 -*-

from urllib.parse import urljoin
from datary.filetrees import DataryFiletrees
from datary.workdirs import DataryWorkdirs
from scrapbag import exclude_empty_values, get_element

@@ -17,3 +17,3 @@

class DataryDatasets(DataryFiletrees):
class DataryDatasets(DataryWorkdirs):
"""

@@ -123,6 +123,6 @@ Datary Datasets module

# retrieve wdir filetree
# retrieve wdir workdir
wdir_filetree = self.get_wdir_filetree(wdir_uuid)
# retrieve last commit filetree
# retrieve last commit workdir
wdir_changes_filetree = self.format_wdir_changes(

@@ -155,3 +155,3 @@ self.get_wdir_changes(wdir_uuid).values())

url = urljoin(self.URL_BASE,
"/workdirs/{}/filetree".format(wdir_uuid))
"/workdirs/{}/workdir".format(wdir_uuid))
params = exclude_empty_values({'pathname': pathname})

@@ -158,0 +158,0 @@ response = self.request(

@@ -124,4 +124,4 @@ # -*- coding: utf-8 -*-

@mock.patch('datary.filetrees.DataryFiletrees.get_wdir_filetree')
@mock.patch('datary.filetrees.DataryFiletrees.get_wdir_changes')
@mock.patch('datary.workdirs.DataryWorkdirs.get_wdir_filetree')
@mock.patch('datary.workdirs.DataryWorkdirs.get_wdir_changes')
def test_get_dataset_uuid(self, mock_get_wdir_changes,

@@ -132,3 +132,3 @@ mock_get_wdir_filetree):

"""
mock_get_wdir_filetree.return_value = self.filetree
mock_get_wdir_filetree.return_value = self.workdir
mock_get_wdir_changes.return_value = self.changes

@@ -151,3 +151,3 @@

# retrive from filetree
# retrive from workdir
path = ''

@@ -154,0 +154,0 @@ basename = 'c'

@@ -73,34 +73,19 @@ # -*- coding: utf-8 -*-

url = urljoin(self.URL_BASE, "workdirs/{}/changes".format(wdir_uuid))
size = element.get('data', {}).get('meta', {}).get('size', 0)
payload = MultipartEncoder({
"blob": (
element.get('basename'),
json.dumps({
'__kern': element.get('data', {}).get('kern'),
'__meta': element.get('data', {}).get('meta'),
}),
'application/json'),
if size >= self._DEFAULT_LIMITED_DATARY_SIZE:
"action": "add",
"filemode": "100644",
"dirname": element.get('path'),
"basename": element.get('basename')
})
payload = MultipartEncoder({
"blob": (
element.get('basename'),
json.dumps({
'__kern': element.get('data', {}).get('kern'),
'__meta': element.get('data', {}).get('meta'),
}),
'application/json'),
self.headers["Content-Type"] = payload.content_type
"action": "add",
"filemode": "100644",
"dirname": element.get('path'),
"basename": element.get('basename')
})
self.headers["Content-Type"] = payload.content_type
else:
self.headers["Content-Type"] = "application/x-www-form-urlencoded"
payload = {
"action": "add",
"filemode": 100644,
"dirname": element.get('path'),
"basename": element.get('basename'),
"kern": json.dumps(element.get('data', {}).get('kern')),
"meta": json.dumps(element.get('data', {}).get('meta'))}
response = self.request(

@@ -107,0 +92,0 @@ url, 'POST', **{'data': payload, 'headers': self.headers})

@@ -8,3 +8,3 @@

from datary.repos import DataryRepos
from datary.filetrees import DataryFiletrees
from datary.workdirs import DataryWorkdirs
from datary.operations.remove import DataryRemoveOperation

@@ -19,3 +19,3 @@ from datary.operations.limits import DataryOperationLimits

class DataryCleanOperation(DataryRemoveOperation, DataryFiletrees,
class DataryCleanOperation(DataryRemoveOperation, DataryWorkdirs,
DataryOperationLimits):

@@ -43,7 +43,7 @@ """

# get filetree
filetree = self.get_wdir_filetree(wdir_uuid)
# get workdir
workdir = self.get_wdir_filetree(wdir_uuid)
# flatten filetree to list
flatten_filetree = flatten(filetree, sep='/')
# flatten workdir to list
flatten_filetree = flatten(workdir, sep='/')

@@ -50,0 +50,0 @@ filetree_keys = [

@@ -44,33 +44,20 @@ # -*- coding: utf-8 -*-

headers = kwargs.get('headers', self.headers)
size = element.get('data', {}).get('meta', {}).get('size', 0)
if size >= self._DEFAULT_LIMITED_DATARY_SIZE:
payload = MultipartEncoder({
"blob": (
element.get('basename'),
json.dumps({
'__kern': element.get('data', {}).get('kern'),
'__meta': element.get('data', {}).get('meta'),
}),
'application/json'),
payload = MultipartEncoder({
"blob": (
element.get('basename'),
json.dumps({
'__kern': element.get('data', {}).get('kern'),
'__meta': element.get('data', {}).get('meta'),
}),
'application/json'),
"action": "modify",
"filemode": "100644",
"dirname": element.get('path'),
"basename": element.get('basename')
})
"action": "modify",
"filemode": "100644",
"dirname": element.get('path'),
"basename": element.get('basename')
})
headers["Content-Type"] = payload.content_type
headers["Content-Type"] = payload.content_type
else:
payload = {
"action": "modify",
"filemode": 100644,
"dirname": element.get('path'),
"basename": element.get('basename'),
"kern": json.dumps(element.get('data', {}).get('kern')),
"meta": json.dumps(element.get('data', {}).get('meta'))}
response = self.request(

@@ -77,0 +64,0 @@ url, 'POST', **{'data': payload, 'headers': headers})

@@ -34,3 +34,3 @@ # -*- coding: utf-8 -*-

"""
content_type = "'Content-Type': 'application/x-www-form-urlencoded'"
content_type = "'Content-Type': 'multipart/form-data;"
mock_request.return_value = MockRequestResponse("")

@@ -57,3 +57,3 @@

# big element
content_type = "'Content-Type': 'multipart/form-data;"
mock_request.return_value = MockRequestResponse("")

@@ -60,0 +60,0 @@

@@ -15,3 +15,3 @@ # -*- coding: utf-8 -*-

@mock.patch('datary.operations.DataryRemoveOperation.delete_file')
@mock.patch('datary.filetrees.DataryFiletrees.get_wdir_filetree')
@mock.patch('datary.workdirs.DataryWorkdirs.get_wdir_filetree')
@mock.patch('datary.operations.DataryRemoveOperation.clear_index')

@@ -25,3 +25,3 @@ @mock.patch('datary.repos.DataryRepos.get_describerepo')

mock_get_describerepo.return_value = self.json_repo
mock_get_wdir_filetree.return_value = self.filetree
mock_get_wdir_filetree.return_value = self.workdir

@@ -42,3 +42,3 @@ self.datary.clean_repo(self.repo_uuid)

# describe repo retrieve None
mock_get_wdir_filetree.return_value = self.filetree
mock_get_wdir_filetree.return_value = self.workdir
mock_get_describerepo.return_value = None

@@ -45,0 +45,0 @@

@@ -5,5 +5,6 @@ # -*- coding: utf-8 -*-

"""
from requests import RequestException
import time
import requests
import structlog
from requests import RequestException

@@ -20,2 +21,3 @@

URL_BASE = "http://api.datary.io/"
tries_limit = 3
headers = {}

@@ -30,4 +32,5 @@

self.headers.update(kwargs.get('headers', {}))
self.tries_limit = kwargs.get('tries_limit', 3)
def request(self, url, http_method, **kwargs):
def request(self, url, http_method, tries=0, **kwargs):
"""

@@ -78,6 +81,36 @@ Sends request to Datary passing config through arguments.

if 199 < content.status_code < 300:
tries = 0
return content
# Check if must wait x seconds returning 429 http code.
elif content.status_code == 429:
# TODO: RETRIEVE TIMESLEEP FROM RETURNED ERR MSG
time_sleep = 2
msg = "Fail Request to datary ({}) - Need to wait {} seconds"
logger.warning(
msg.format(content.status_code, time_sleep),
url=url, http_method=http_method,
code=content.status_code,
text=content.text,
# kwargs=kwargs,
)
time.sleep(time_sleep)
tries += 1
if tries >= self.tries_limit:
logger.error(
"Request Tries Limit Exceeded!!",
url=url,
http_method=http_method,
# requests_args=kwargs,
)
else:
return self.request(url, http_method, tries, **kwargs)
else:
msg = "Fail Request to datary done with code {}"
tries = 0
logger.error(

@@ -93,2 +126,3 @@ msg.format(content.status_code),

except RequestException as ex:
tries = 0
logger.error(

@@ -95,0 +129,0 @@ "Fail request to Datary - {}".format(ex),

@@ -18,3 +18,4 @@ # -*- coding: utf-8 -*-

@mock.patch('datary.requests.requests.requests')
def test_request(self, mock_requests):
@mock.patch('datary.requests.requests.time')
def test_request(self, mock_time, mock_requests):
"""

@@ -65,2 +66,32 @@ Test get_request

# test status code 429
responses_429_test = [
MockRequestResponse(
"Request limit exceeded, must wait for XX seconds",
status_code=429),
MockRequestResponse(
"Request limit exceeded, must wait for XX seconds",
status_code=429),
MockRequestResponse(
"Request limit exceeded, must wait for XX seconds",
status_code=429),
MockRequestResponse(
"Everything OK",
status_code=200)
]
mock_requests.get.side_effect = iter(responses_429_test)
result7 = self.datary.request(self.url, 'GET')
self.assertEqual(result7, None)
# upgrade tries limit
self.datary.tries_limit = 4
mock_requests.get.side_effect = iter(responses_429_test)
result8 = self.datary.request(self.url, 'GET')
self.assertEqual(result8.text, 'Everything OK')
self.assertEqual(mock_time.sleep.call_count, 3+3)
mock_requests.get.side_effect = requests.RequestException('err')

@@ -70,4 +101,5 @@ result6 = self.datary.request(self.url, 'GET')

self.assertEqual(mock_requests.get.call_count, 3)
self.assertEqual(mock_requests.get.call_count, 10)
self.assertEqual(mock_requests.post.call_count, 1)
self.assertEqual(mock_requests.put.call_count, 1)
self.assertEqual(mock_requests.delete.call_count, 1)

@@ -114,3 +114,3 @@ # -*- coding: utf-8 -*-

"addedElements": []},
"filetree": {
"workdir": {
"file_test1": "3a26a47b6e7f28c77380eccc8aec23sd6dc0201e",

@@ -148,3 +148,3 @@ "folder_test1": {

filetree = {
workdir = {
'__self': '__self_sha1',

@@ -151,0 +151,0 @@ 'a': 'a_sha1',

@@ -5,2 +5,2 @@ #!/usr/bin/env python

"""
__version__ = "0.1.28"
__version__ = "0.1.29"
Metadata-Version: 1.1
Name: datary
Version: 0.1.28
Version: 0.1.29
Summary: Datary Python sdk lib

@@ -5,0 +5,0 @@ Home-page: https://github.com/Datary/python-sdk

# -*- coding: utf-8 -*-
"""
Datary Filetrees Module
"""
from .filetrees import DataryFiletrees
# -*- coding: utf-8 -*-
"""
Datary sdk Filetrees File
"""
import os
from urllib.parse import urljoin
from datary.repos import DataryRepos
from scrapbag import force_list, add_element
import structlog
logger = structlog.getLogger(__name__)
class DataryFiletrees(DataryRepos):
"""
Datary Filetrees module class
"""
def get_commit_filetree(self, repo_uuid, commit_sha1):
"""
============== ============= ====================================
Parameter Type Description
============== ============= ====================================
repo_uuid int repository id
commit_sha1 str filetree sha1
============== ============= ====================================
Returns:
filetree of all commits done in a repo.
"""
url = urljoin(self.URL_BASE,
"commits/{}/filetree".format(commit_sha1))
params = {'namespace': repo_uuid}
response = self.request(
url, 'GET', **{'headers': self.headers, 'params': params})
return response.json() if response else {}
def get_wdir_filetree(self, wdir_uuid):
"""
============== ============= ====================================
Parameter Type Description
============== ============= ====================================
wdir_uuid str working directory id
============== ============= ====================================
Returns:
filetree of a repo workdir.
"""
url = urljoin(self.URL_BASE,
"workdirs/{}/filetree".format(wdir_uuid))
response = self.request(url, 'GET', **{'headers': self.headers})
return response.json() if response else {}
def get_wdir_changes(self, wdir_uuid=None, **kwargs):
"""
================ ============= ====================================
Parameter Type Description
================ ============= ====================================
wdir_uuid str working directory id
================ ============= ====================================
Returns:
(dict) changes in workdir.
"""
# try to take wdir_uuid with kwargs
if not wdir_uuid:
wdir_uuid = self.get_describerepo(
**kwargs).get('workdir', {}).get('uuid')
url = urljoin(self.URL_BASE,
"workdirs/{}/changes".format(wdir_uuid))
response = self.request(url, 'GET', **{'headers': self.headers})
return response.json() if response else {}
def format_wdir_changes(self, wdir_changes_tree):
"""
================== ============= ==================================
Parameter Type Description
================== ============= ==================================
wdir_changes_tree list working changes tree
================== ============= ==================================
Returns:
(dict) changes in workdir formatting as filetree format.
"""
result = {}
for sublist in list(wdir_changes_tree):
for item in force_list(sublist):
add_element(
result,
os.path.join(item.get('dirname', ''),
item.get('basename', '')),
item.get('inode', 'unkwown_dataset_uuid')
)
return result
# -*- coding: utf-8 -*-
"""
Datary python sdk Filetrees test file
"""
import mock
from datary.test.test_datary import DataryTestCase
from datary.test.mock_requests import MockRequestResponse
class DataryFiletreesTestCase(DataryTestCase):
"""
Datary Filetrees Test Case
"""
@mock.patch('datary.Datary.request')
def test_get_commit_filetree(self, mock_request):
"""
Test get_commit_filetree
"""
mock_request.return_value = MockRequestResponse(
"", json=self.wdir_json.get('filetree'))
filetree = self.datary.get_commit_filetree(
self.repo_uuid, self.commit_sha1)
self.assertEqual(mock_request.call_count, 1)
assert isinstance(filetree, dict)
@mock.patch('datary.Datary.request')
def test_get_wdir_filetree(self, mock_request):
"""
Test get_wdir_filetree
"""
mock_request.return_value = MockRequestResponse(
"", json=self.wdir_json.get('changes'))
changes = self.datary.get_wdir_filetree(self.repo_uuid)
self.assertEqual(mock_request.call_count, 1)
assert isinstance(changes, dict)
@mock.patch('datary.Datary.request')
@mock.patch('datary.repos.DataryRepos.get_describerepo')
def test_get_wdir_changes(self, mock_describerepo, mock_request):
"""
Test get_wdir_changes
"""
mock_request.return_value = MockRequestResponse(
"", json=self.wdir_json.get('filetree'))
filetree = self.datary.get_wdir_changes(self.wdir_uuid)
self.assertEqual(mock_request.call_count, 1)
assert isinstance(filetree, dict)
mock_describerepo.return_value = self.json_repo
mock_request.return_value = MockRequestResponse(
"", json=self.wdir_json.get('filetree'))
filetree = self.datary.get_wdir_changes(repo_uuid=self.repo_uuid)
self.assertEqual(mock_request.call_count, 2)
assert isinstance(filetree, dict)
def test_format_wdir_change(self):
"""
Test format_wdir_change
"""
format_filetree = self.datary.format_wdir_changes
treeformated_changes = format_filetree(self.changes.values())
self.assertEqual(len(treeformated_changes.keys()), 3)
self.assertEqual(treeformated_changes.get('a'), 'inode2_changes')
self.assertEqual(treeformated_changes.get('b'),
{'bb': 'inode1_changes'})
self.assertEqual(treeformated_changes.get('d'), 'inode3_changes')