easycloudapi - pypi Package Compare versions

Comparing version 0.0.2 to 0.0.3

src/easycloudapi/gcp/bigquery/__init__.py

src/easycloudapi/gcp/bigquery/bigquery_schema.py

+import json
+
+# Creating a JSON schema file
+# Ref: https://cloud.google.com/bigquery/docs/schemas#creating_a_JSON_schema_file
+# test:
+# 1. https://transform.tools/json-to-big-query
+# 2. https://bigquery-json-schema-generator.com/
+
+
+class BigquerySchema:
+    # get generic bq coloum schema
+    @staticmethod
+    def _get_generic_coloum_schema(col_name: str, col_type="STRING", col_mode="NULLABLE"):
+        """
+        generic bigquery column schema
+        """
+        generic_coloum_schema = {}
+        generic_coloum_schema["name"] = col_name
+        generic_coloum_schema["type"] = col_type
+        if col_mode != "NULLABLE":
+            generic_coloum_schema["mode"] = col_mode
+        # generic_coloum_schema["description"] = "_" + col_name
+        return generic_coloum_schema
+
+    # generate bigquery schema
+    @staticmethod
+    def generate_bq_schema(data: dict):
+        """
+        generate bigquery schema
+        """
+        out_schema = []
+        for i, j in data.items():
+            if type(j) == str:
+                out_schema.append(
+                    BigquerySchema._get_generic_coloum_schema(col_name=str(i))
+                )
+            elif type(j) == int:
+                out_schema.append(
+                    BigquerySchema._get_generic_coloum_schema(col_name=str(i), col_type="INTEGER")
+                )
+            elif type(j) == list:
+                element = j[0]
+                if type(element) == str:
+                    out_schema.append(
+                        BigquerySchema._get_generic_coloum_schema(col_name=str(i),
+                                                                    col_mode="repeated")
+                    )
+                elif type(element) == int:
+                    out_schema.append(
+                        BigquerySchema._get_generic_coloum_schema(col_name=str(i),
+                                                                    col_type="INTEGER",
+                                                                    col_mode="repeated")
+                    )
+                elif type(element) == dict:
+                    dict_obj = BigquerySchema._get_generic_coloum_schema(col_name=str(i),
+                                                                           col_type="record")
+                    dict_obj["fields"] = BigquerySchema.generate_bq_schema(element)
+                    out_schema.append(dict_obj)
+            elif type(j) == dict:
+                dict_obj = BigquerySchema._get_generic_coloum_schema(col_name=str(i),
+                                                                       col_type="record")
+                dict_obj["fields"] = BigquerySchema.generate_bq_schema(j)
+                out_schema.append(dict_obj)
+            else:
+                pass
+        return out_schema
+
+
+if __name__ == "__main__":
+    bs = BigquerySchema()
+    with open("tests//test_data//sample_json_01.json") as json_file:
+        sample_dict = json.load(json_file)
+    out = bs.generate_bq_schema(data=sample_dict)
+    print(f"output from generate_bq_schema:\n{out}")

src/easycloudapi/generic/datetime/__init__.py

src/easycloudapi/generic/datetime/generate_date_dimention.py

+import pandas as pd
+from datetime import datetime as dt
+from datetime import timedelta
+from datetime import date
+
+
+def generate_date_dimension(start_date: str = "", end_date: str = ""):
+
+    start_date = start_date.replace("/", "-").replace("\\", "-")
+    end_date = end_date.replace("/", "-").replace("\\", "-")
+
+    # cur_date = dt.today().strftime("%Y-%m-%d")  # "%Y-%m-%d" "%d-%m-%Y"
+    prev_date = (dt.today() - timedelta(days=7)).strftime("%Y-%m-%d")
+    next_date = (dt.today() + timedelta(days=7)).strftime("%Y-%m-%d")
+
+    if start_date is None or start_date == "":
+        start_date = prev_date
+    if end_date is None or end_date == "":
+        end_date = next_date
+
+    _start_date = dt.strptime(start_date, "%Y-%m-%d").strftime("%Y-%m-%d")
+    _end_date = dt.strptime(end_date, "%Y-%m-%d").strftime("%Y-%m-%d")
+
+    data = dict()
+    # data["current_date"] = cur_date
+    data["start_date"] = _start_date
+    data["end_date"] = _end_date
+
+    # empty dataframe
+    # df_empty = pd.DataFrame()
+
+    # use date functionality of pandas dataframe
+    date_df = pd.DataFrame({"date_time": pd.date_range(start=_start_date, end=_end_date)})
+
+    # date_df.date_time is pandas series with "date_time" column
+    date_df["date"] = date_df.date_time.dt.strftime("%Y-%m-%d")
+    date_df["day"] = date_df.date_time.dt.day
+    date_df["week_day"] = date_df.date_time.dt.weekday
+    date_df["week_name"] = date_df.date_time.dt.day_name()
+    date_df["week_of_year"] = date_df.date_time.dt.isocalendar().week
+    date_df["month"] = date_df.date_time.dt.month
+    date_df["quarter"] = date_df.date_time.dt.quarter
+    date_df["year"] = date_df.date_time.dt.year
+
+    # adding date_id column as unique id at 1st position
+    date_df.insert(1, "date_id", (date_df.year.astype(str) +
+                                  date_df.month.astype(str).str.zfill(2) +
+                                  date_df.day.astype(str).str.zfill(2)).astype(int))
+
+    # convert pandas dataframe to python dict
+    data["date_dimension"] = str(date_df.loc[:, "date_id":"year"].to_dict())
+
+    return data
+
+
+if __name__ == "__main__":
+    out1 = generate_date_dimension(start_date="2023\\08\\01", end_date="2023\\08\\03")
+    out2 = generate_date_dimension(start_date="2023/08/01", end_date="2023/08/03")
+    out3 = generate_date_dimension(start_date="2023-08-01", end_date="2023-08-03")
+    print(f"out1: {out1}")
+    print(f"out2: {out2}")
+    print(f"out3: {out3}")

LICENSE

@@ -1,2 +0,2 @@
-MIT License
+## MIT License
 
@@ -3,0 +3,0 @@ Copyright (c) 2023 easycloudapi

PKG-INFO

 Metadata-Version: 2.1
 Name: easycloudapi
-Version: 0.0.2
+Version: 0.0.3
 Summary: Providing utility functions to help other python developers
@@ -81,4 +81,7 @@ Home-page: https://github.com/easycloudapi/python_utility
 1. Refer to [Developer Guide](developer.md)
-MIT License
 
+
+
+## MIT License
+
 Copyright (c) 2023 easycloudapi
@@ -85,0 +88,0 @@

README.md

@@ -63,2 +63,4 @@ # easycloudapi
 ## How to Contribute
-1. Refer to [Developer Guide](developer.md)
+1. Refer to [Developer Guide](developer.md)
+
+

setup.cfg

 [metadata]
 name = easycloudapi
-version = 0.0.2
+version = 0.0.3
 author = Indranil Pal
@@ -5,0 +5,0 @@ author_email = [e-mail]

src/easycloudapi.egg-info/PKG-INFO

 Metadata-Version: 2.1
 Name: easycloudapi
-Version: 0.0.2
+Version: 0.0.3
 Summary: Providing utility functions to help other python developers
@@ -81,4 +81,7 @@ Home-page: https://github.com/easycloudapi/python_utility
 1. Refer to [Developer Guide](developer.md)
-MIT License
 
+
+
+## MIT License
+
 Copyright (c) 2023 easycloudapi
@@ -85,0 +88,0 @@

src/easycloudapi.egg-info/SOURCES.txt

@@ -13,3 +13,7 @@ LICENSE
 src/easycloudapi/gcp/__init__.py
+src/easycloudapi/gcp/bigquery/__init__.py
+src/easycloudapi/gcp/bigquery/bigquery_schema.py
 src/easycloudapi/generic/__init__.py
+src/easycloudapi/generic/datetime/__init__.py
+src/easycloudapi/generic/datetime/generate_date_dimention.py
 tests/test.py

tests/test.py

@@ -1,13 +0,19 @@
-import json
-from easycloudapi.generic.datetime.generate_date_dimention import generate_date_dimension
+from easycloudapi.gcp.bigquery.bigquery_schema import BigquerySchema
 
+sample_json = {
+    "name": "Danilo",
+    "age": 32,
+    "date_joined": "2020-11-05",
+    "location": {"country": "United Kingdom", "city": "London"},
+    "years_active": [2020, 2021, 2022],
+    "favourite_movies": [
+        {"name": "Momento", "year": "2000"},
+        {"name": "Se7en", "year": "1995"},
+        {"name": "Momento", "year": "2000"}
+    ]
+}
 
-out1 = generate_date_dimension(start_date="2023\\08\\01", end_date="2023\\08\\03")
-print(f"out1: {out1}")
+bq_schema_obj = BigquerySchema()
+out_generate_bq_schema = bq_schema_obj.generate_bq_schema(data=sample_json)
 
-out2 = generate_date_dimension(start_date="2023/08/01", end_date="2023/08/03")
-print(f"out2: {out2}")
-
-out3 = generate_date_dimension(start_date="2023-08-01", end_date="2023-08-03")
-print(f"out3: {out3}")
-
+print(f"out_generate_bq_schema:\n{out_generate_bq_schema}")

Alert delta unavailable

Currently unable to show alert delta for PyPI packages.

Improved metrics

Total package byte prevSize

18936

46.98%

Number of package files

19

26.67%

Lines of code

131

1537.5%