geeup - npm Package Compare versions

Comparing version 0.5.8 to 0.5.9

geeup.egg-info/PKG-INFO

 Metadata-Version: 2.1
 Name: geeup
-Version: 0.5.8
+Version: 0.5.9
 Summary: Simple Client for Earth Engine Uploads
@@ -13,3 +13,3 @@ Home-page: https://github.com/samapriya/geeup
         [![Hits-of-Code](https://hitsofcode.com/github/samapriya/geeup?branch=master)](https://hitsofcode.com/github/samapriya/geeup?branch=master)
-        [![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.6665370.svg)](https://doi.org/10.5281/zenodo.6665370)
+        [![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.7047124.svg)](https://doi.org/10.5281/zenodo.7047124)
         [![PyPI version](https://badge.fury.io/py/geeup.svg)](https://badge.fury.io/py/geeup)
@@ -19,2 +19,4 @@ [![Downloads](https://pepy.tech/badge/geeup/month)](https://pepy.tech/project/geeup)
         ![CI geeup](https://github.com/samapriya/geeup/workflows/CI%20geeup/badge.svg)
+        [![Donate](https://img.shields.io/badge/Donate-Buy%20me%20a%20Chai-teal)](https://www.buymeacoffee.com/samapriya)
+        [![](https://img.shields.io/static/v1?label=Sponsor&message=%E2%9D%A4&logo=GitHub&color=%23fe8e86)](https://github.com/sponsors/samapriya)
         
@@ -26,4 +28,4 @@ This tool came from the simple need to handle batch uploads of both image assets to collections. Thanks to the new table feature, the possibility of batch uploading shapefiles and CSVs into a folder became more common. This tool was designed to allow the user to preprocess imagery and shapefiles and process all formats of uploads currently possible on Google Earth Engine. The command line tool provides a simple quota query tool and a task state tool, to name a few additional tasks you can run with this tool. I hope this tool with a simple objective proves helpful to a few users of Google Earth Engine.
         ```
-        Samapriya Roy. (2022). samapriya/geeup: geeup: Simple CLI for Earth Engine Uploads (0.5.6).
-        Zenodo. https://doi.org/10.5281/zenodo.6665370
+        Samapriya Roy. (2022). samapriya/geeup: geeup: Simple CLI for Earth Engine Uploads (0.5.8).
+        Zenodo. https://doi.org/10.5281/zenodo.7047124
         ```
@@ -30,0 +32,0 @@

geeup.egg-info/requires.txt

@@ -0,1 +1,2 @@
+wheel
 earthengine_api>=0.1.274
@@ -2,0 +3,0 @@ logzero>=1.5.0

geeup/__init__.py

@@ -5,2 +5,2 @@ # -*- coding: utf-8 -*-
 __email__ = "[e-mail]"
-__version__ = "0.5.8"
+__version__ = "0.5.9"

geeup/batch_uploader.py

@@ -94,3 +94,3 @@ from __future__ import print_function
     nodata_value=None,
-    bucket_name=None,
+    overwrite=None,
 ):
@@ -124,3 +124,3 @@ schema = {"collection_path": {
     images_for_upload_path = __find_remaining_assets_for_upload(
-        all_images_paths, destination_path
+        all_images_paths, destination_path, overwrite
     )
@@ -132,7 +132,7 @@ no_images = len(images_for_upload_path)
         sys.exit(1)
-
+    file_count = len(images_for_upload_path)
     for current_image_no, image_path in enumerate(natsorted(images_for_upload_path)):
-        logging.info(
-            f"Processing image {current_image_no + 1} out of {no_images} : {image_path}"
-        )
+        # logging.info(
+        #     f"Processing image {current_image_no + 1} out of {no_images} : {image_path}"
+        # )
         status = ["RUNNING", "PENDING"]
@@ -248,9 +248,15 @@ task_count = len(
                             json.dump(main_payload, outfile)
-                        subprocess.call(
-                            "earthengine upload image --manifest "
-                            + '"'
-                            + os.path.join(lp, "data.json")
-                            + '"',
-                            shell=True,
-                            stdout=subprocess.PIPE,
+                        check_list = ['yes', 'y']
+                        if overwrite is not None and overwrite.lower() in check_list:
+                            output = subprocess.check_output(
+                                f"earthengine upload image --manifest {os.path.join(lp, 'data.json')} -f",
+                                shell=True
+                            )
+                        else:
+                            output = subprocess.check_output(
+                                f"earthengine upload image --manifest {os.path.join(lp, 'data.json')}",
+                                shell=True
+                            )
+                        logging.info(
+                            f"Ingesting {i+1} of {file_count} {str(os.path.basename(asset_full_path))} with Task Id: {output.decode('ascii').strip().split(' ')[-1]}"
                         )
@@ -264,40 +270,44 @@ except Exception as error:
 
-def __find_remaining_assets_for_upload(path_to_local_assets, path_remote):
+def __find_remaining_assets_for_upload(path_to_local_assets, path_remote, overwrite):
     local_assets = [__get_filename_from_path(
         path) for path in path_to_local_assets]
     if __collection_exist(path_remote):
-        remote_assets = __get_asset_names_from_collection(path_remote)
-        tasked_assets = []
-        status = ["RUNNING", "PENDING"]
-        for task in ee.data.listOperations():
-            if (
-                task["metadata"]["type"] == "INGEST_IMAGE"
-                and task["metadata"]["state"] in status
-            ):
-                tasked_assets.append(
-                    task["metadata"]["description"]
-                    .split(":")[-1]
-                    .split("/")[-1]
-                    .replace('"', "")
+        check_list = ['yes', 'y']
+        if overwrite is not None and overwrite.lower() in check_list:
+            return path_to_local_assets
+        else:
+            remote_assets = __get_asset_names_from_collection(path_remote)
+            tasked_assets = []
+            status = ["RUNNING", "PENDING"]
+            for task in ee.data.listOperations():
+                if (
+                    task["metadata"]["type"] == "INGEST_IMAGE"
+                    and task["metadata"]["state"] in status
+                ):
+                    tasked_assets.append(
+                        task["metadata"]["description"]
+                        .split(":")[-1]
+                        .split("/")[-1]
+                        .replace('"', "")
+                    )
+            if len(remote_assets) >= 0:
+                assets_left_for_upload = set(local_assets).difference(
+                    set(remote_assets), set(tasked_assets)
                 )
-        if len(remote_assets) >= 0:
-            assets_left_for_upload = set(local_assets).difference(
-                set(remote_assets), set(tasked_assets)
-            )
-            if len(assets_left_for_upload) == 0:
-                print(
-                    f"All assets already ingested or running : {len(set(remote_assets))} assets ingested with {len(set(tasked_assets))} tasks running or submitted"
-                )
-                sys.exit(1)
-            elif len(assets_left_for_upload) > 0:
-                print(
-                    f"Total of {len(assets_left_for_upload)} assets remaining : Total of {len(set(remote_assets))} already in collection with {len(set(tasked_assets))} associated tasks running or submitted"
-                )
+                if len(assets_left_for_upload) == 0:
+                    print(
+                        f"All assets already ingested or running : {len(set(remote_assets))} assets ingested with {len(set(tasked_assets))} tasks running or submitted"
+                    )
+                    sys.exit(1)
+                elif len(assets_left_for_upload) > 0:
+                    print(
+                        f"Total of {len(assets_left_for_upload)} assets remaining : Total of {len(set(remote_assets))} already in collection with {len(set(tasked_assets))} associated tasks running or submitted"
+                    )
 
-            assets_left_for_upload_full_path = [
-                path
-                for path in path_to_local_assets
-                if __get_filename_from_path(path) in assets_left_for_upload
-            ]
-            return assets_left_for_upload_full_path
+                assets_left_for_upload_full_path = [
+                    path
+                    for path in path_to_local_assets
+                    if __get_filename_from_path(path) in assets_left_for_upload
+                ]
+                return assets_left_for_upload_full_path
 
@@ -422,14 +432,2 @@ return path_to_local_assets
 )
-def __upload_file_gcs(storage_client, bucket_name, image_path):
-    bucket = storage_client.get_bucket(bucket_name)
-    blob_name = __get_filename_from_path(path=image_path)
-    blob = bucket.blob(blob_name)
-
-    blob.upload_from_filename(image_path)
-
-    url = "gs://" + bucket_name + "/" + blob_name
-
-    return url
-
-
 def __get_filename_from_path(path):
@@ -443,15 +441,2 @@ return os.path.splitext(os.path.basename(os.path.normpath(path)))[0]
 
-def __wait_for_tasks_to_complete(waiting_time, no_allowed_tasks_running):
-    tasks_running = __get_number_of_running_tasks()
-    while tasks_running > no_allowed_tasks_running:
-        logging.info(
-            "Number of running tasks is %d. Sleeping for %d s until it goes down to %d",
-            tasks_running,
-            waiting_time,
-            no_allowed_tasks_running,
-        )
-        time.sleep(waiting_time)
-        tasks_running = __get_number_of_running_tasks()
-
-
 def __collection_exist(path):
@@ -458,0 +443,0 @@ return True if ee.data.getInfo(path) else False

geeup/geeup.py

@@ -49,3 +49,2 @@ from .batch_uploader import upload
         import pipwin
-
         if pipwin.__version__ == "0.5.0":
@@ -55,3 +54,3 @@ pass
             subprocess.call(
-                "python" + str(version) + " -m pip install pipwin==0.5.0", shell=True
+                f"{sys.executable}" + " -m pip install pipwin==0.5.0", shell=True
             )
@@ -64,9 +63,5 @@ subprocess.call("pipwin refresh", shell=True)
         fullpath = os.path.join(home_dir, ".pipwin")
-        file_mod_time = os.stat(fullpath).st_mtime
-        if int((time.time() - file_mod_time) / 60) > 90000:
-            print("Refreshing your pipwin cache")
-            subprocess.call("pipwin refresh", shell=True)
     except ImportError:
         subprocess.call(
-            "python" + str(version) + " -m pip install pipwin==0.5.0", shell=True
+            f"{sys.executable}" + " -m pip install pipwin==0.5.0", shell=True
         )
@@ -90,29 +85,7 @@ subprocess.call("pipwin refresh", shell=True)
     except ImportError:
-        subprocess.call("pipwin install pandas", shell=True)
+        subprocess.call(
+            f"{sys.executable}" + " -m pip install pandas", shell=True
+        )
     except Exception as e:
         logger.exception(e)
-    try:
-        import pyproj
-    except ImportError:
-        subprocess.call("pipwin install pyproj", shell=True)
-    except Exception as e:
-        logger.exception(e)
-    try:
-        import shapely
-    except ImportError:
-        subprocess.call("pipwin install shapely", shell=True)
-    except Exception as e:
-        logger.exception(e)
-    try:
-        import fiona
-    except ImportError:
-        subprocess.call("pipwin install fiona", shell=True)
-    except Exception as e:
-        logger.exception(e)
-    try:
-        import geopandas
-    except ImportError:
-        subprocess.call("pipwin install geopandas", shell=True)
-    except Exception as e:
-        logger.exception(e)
 
@@ -334,17 +307,2 @@ lpath = os.path.dirname(os.path.realpath(__file__))
 
-def _comma_separated_strings(string):
-    """Parses an input consisting of comma-separated strings.
-    Slightly modified version of: https://pypkg.com/pypi/earthengine-api/f/ee/cli/commands.py
-    """
-    error_msg = (
-        "Argument should be a comma-separated list of alphanumeric strings (no spaces or other"
-        "special characters): {}"
-    )
-    values = string.split(",")
-    for name in values:
-        if not name.isalnum():
-            raise argparse.ArgumentTypeError(error_msg.format(string))
-    return values
-
-
 def upload_from_parser(args):
@@ -358,2 +316,3 @@ upload(
         pyramiding=args.pyramids,
+        overwrite=args.overwrite
     )
@@ -369,2 +328,3 @@
         y=args.y,
+        overwrite=args.overwrite
     )
@@ -620,2 +580,6 @@
     )
+    optional_named.add_argument(
+        "--overwrite",
+        help="Default is No but you can pass yes or y",
+    )
     required_named.add_argument(
@@ -655,2 +619,6 @@ "-u", "--user", help="Google account name (gmail address)."
     )
+    optional_named.add_argument(
+        "--overwrite",
+        help="Default is No but you can pass yes or y",
+    )
     parser_tabup.set_defaults(func=tabup_from_parser)
@@ -657,0 +625,0 @@

geeup/tuploader.py

@@ -131,3 +131,3 @@ __copyright__ = """
 
-def tabup(dirc, uname, destination, x, y):
+def tabup(dirc, uname, destination, x, y, overwrite=None):
     ee.Initialize()
@@ -187,4 +187,8 @@ schema = {"folder_path": {
             )
-    diff_set = set(table_exists).difference(
-        set(gee_table_exists), set(tasked_assets))
+    check_list = ['yes', 'y']
+    if overwrite is not None and overwrite.lower() in check_list:
+        diff_set = set(table_exists)
+    else:
+        diff_set = set(table_exists).difference(
+            set(gee_table_exists), set(tasked_assets))
     if len(diff_set) > 0:
@@ -274,9 +278,12 @@ print(
                                     json.dump(main_payload, outfile)
-                                output = subprocess.check_output(
-                                    "earthengine upload table --manifest "
-                                    + '"'
-                                    + os.path.join(lp, "data.json")
-                                    + '"',
-                                    shell=True,
-                                )
+                                if overwrite is not None and overwrite.lower() in check_list:
+                                    output = subprocess.check_output(
+                                        f"earthengine upload table --manifest {os.path.join(lp, 'data.json')} -f",
+                                        shell=True
+                                    )
+                                else:
+                                    output = subprocess.check_output(
+                                        f"earthengine upload table --manifest {os.path.join(lp, 'data.json')}",
+                                        shell=True
+                                    )
                                 logging.info(
@@ -283,0 +290,0 @@ f"Ingesting {i+1} of {file_count} {str(os.path.basename(asset_full_path))} with Task Id: {output.decode('ascii').strip().split(' ')[-1]}"

PKG-INFO

 Metadata-Version: 2.1
 Name: geeup
-Version: 0.5.8
+Version: 0.5.9
 Summary: Simple Client for Earth Engine Uploads
@@ -13,3 +13,3 @@ Home-page: https://github.com/samapriya/geeup
         [![Hits-of-Code](https://hitsofcode.com/github/samapriya/geeup?branch=master)](https://hitsofcode.com/github/samapriya/geeup?branch=master)
-        [![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.6665370.svg)](https://doi.org/10.5281/zenodo.6665370)
+        [![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.7047124.svg)](https://doi.org/10.5281/zenodo.7047124)
         [![PyPI version](https://badge.fury.io/py/geeup.svg)](https://badge.fury.io/py/geeup)
@@ -19,2 +19,4 @@ [![Downloads](https://pepy.tech/badge/geeup/month)](https://pepy.tech/project/geeup)
         ![CI geeup](https://github.com/samapriya/geeup/workflows/CI%20geeup/badge.svg)
+        [![Donate](https://img.shields.io/badge/Donate-Buy%20me%20a%20Chai-teal)](https://www.buymeacoffee.com/samapriya)
+        [![](https://img.shields.io/static/v1?label=Sponsor&message=%E2%9D%A4&logo=GitHub&color=%23fe8e86)](https://github.com/sponsors/samapriya)
         
@@ -26,4 +28,4 @@ This tool came from the simple need to handle batch uploads of both image assets to collections. Thanks to the new table feature, the possibility of batch uploading shapefiles and CSVs into a folder became more common. This tool was designed to allow the user to preprocess imagery and shapefiles and process all formats of uploads currently possible on Google Earth Engine. The command line tool provides a simple quota query tool and a task state tool, to name a few additional tasks you can run with this tool. I hope this tool with a simple objective proves helpful to a few users of Google Earth Engine.
         ```
-        Samapriya Roy. (2022). samapriya/geeup: geeup: Simple CLI for Earth Engine Uploads (0.5.6).
-        Zenodo. https://doi.org/10.5281/zenodo.6665370
+        Samapriya Roy. (2022). samapriya/geeup: geeup: Simple CLI for Earth Engine Uploads (0.5.8).
+        Zenodo. https://doi.org/10.5281/zenodo.7047124
         ```
@@ -30,0 +32,0 @@

README.md

@@ -5,3 +5,3 @@ # geeup: Simple CLI for Earth Engine Uploads
 [![Hits-of-Code](https://hitsofcode.com/github/samapriya/geeup?branch=master)](https://hitsofcode.com/github/samapriya/geeup?branch=master)
-[![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.6665370.svg)](https://doi.org/10.5281/zenodo.6665370)
+[![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.7047124.svg)](https://doi.org/10.5281/zenodo.7047124)
 [![PyPI version](https://badge.fury.io/py/geeup.svg)](https://badge.fury.io/py/geeup)
@@ -11,2 +11,4 @@ [![Downloads](https://pepy.tech/badge/geeup/month)](https://pepy.tech/project/geeup)
 ![CI geeup](https://github.com/samapriya/geeup/workflows/CI%20geeup/badge.svg)
+[![Donate](https://img.shields.io/badge/Donate-Buy%20me%20a%20Chai-teal)](https://www.buymeacoffee.com/samapriya)
+[![](https://img.shields.io/static/v1?label=Sponsor&message=%E2%9D%A4&logo=GitHub&color=%23fe8e86)](https://github.com/sponsors/samapriya)
 
@@ -18,4 +20,4 @@ This tool came from the simple need to handle batch uploads of both image assets to collections. Thanks to the new table feature, the possibility of batch uploading shapefiles and CSVs into a folder became more common. This tool was designed to allow the user to preprocess imagery and shapefiles and process all formats of uploads currently possible on Google Earth Engine. The command line tool provides a simple quota query tool and a task state tool, to name a few additional tasks you can run with this tool. I hope this tool with a simple objective proves helpful to a few users of Google Earth Engine.
 ```
-Samapriya Roy. (2022). samapriya/geeup: geeup: Simple CLI for Earth Engine Uploads (0.5.6).
-Zenodo. https://doi.org/10.5281/zenodo.6665370
+Samapriya Roy. (2022). samapriya/geeup: geeup: Simple CLI for Earth Engine Uploads (0.5.8).
+Zenodo. https://doi.org/10.5281/zenodo.7047124
 ```
@@ -22,0 +24,0 @@

setup.py

@@ -18,6 +18,7 @@ import os
     name="geeup",
-    version="0.5.8",
+    version="0.5.9",
     packages=find_packages(),
     url="https://github.com/samapriya/geeup",
     install_requires=[
+        "wheel",
         "earthengine_api>=0.1.274",
@@ -24,0 +25,0 @@ "logzero>=1.5.0",

Alert delta unavailable

Currently unable to show alert delta for PyPI packages.

Improved metrics

Total package byte prevSize

149795

0.15%

Worsened metrics

Lines of code

1563

-1.76%