fmdt-python - pypi Package Compare versions

Comparing version 0.0.41 to 0.0.42

fmdt_python-0.0.42.dist-info/LICENSE

+MIT License
+
+Copyright (c) 2023 Evan Voyles
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

fmdt_python-0.0.42.dist-info/METADATA

+Metadata-Version: 2.1
+Name: fmdt-python
+Version: 0.0.42
+Summary: utlity functions for fast meteor detection toolbox
+Author-email: Evan Voyles <[e-mail]>
+Project-URL: fmdt, https://github.com/alsoc/fmdt
+Project-URL: Homepage, https://github.com/ejovo13/fmdt_scripts
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: numpy
+Requires-Dist: ffmpeg-python
+Requires-Dist: requests
+Requires-Dist: termcolor
+Requires-Dist: matplotlib
+Requires-Dist: pandas
+Requires-Dist: jsonpickle
+Requires-Dist: appdirs
+Requires-Dist: Deprecated
+
+A series of Python scripts to facilitate the processing of [fmdt's](https://github.com/alsoc/fmdt) output
+
+Scripts for video editing rely on [ffmpeg-python's](https://github.com/kkroening/ffmpeg-python) simple Python bindings to ffmpeg. Make sure you install `ffmpeg-python` before trying any of the video editing functionality.
+
+### Installation
+
+```
+pip install fmdt-python
+```
+
+`fmdt/core.py` should contain the functions that are called directly in scripts.
+`fmdt/utils.py` contains utility functions that `fmdt.core` makes use of.
+
+Example to split a video using tracking information already provided by `fmdt-detect`:
+
+```
+import fmdt
+
+fmdt.split_video_at_meteors("demo.mp4", "ex1_detect_tracks.txt")
+```
+
+#### TODO
+
+- [x] Upload fmdt to pip so that we can download fmdt and call scripts from anywhere
+- [x] Add API to call fmdt executables like `fmdt-detect` and `fmdt-visu`

fmdt_python-0.0.42.dist-info/RECORD

+fmdt/__init__.py,sha256=G5m3lwVytqLgLlDKcT95fwZbL8cE0T4VZ3b_R6KR06c,1624
+fmdt/__test__.py,sha256=FXbpPO1x5EatXgegxcJLGb1iUpWMCz9xuSvOWc0zr8A,1919
+fmdt/api.py,sha256=rFHKXzmOnQnoXUJC13QjbXYXfwFZdmVZ1NnQzX52cH8,12273
+fmdt/args.py,sha256=ghPCc_H2-O5QEXjdfapmj9alFZcBIE_WCWEIiEh7Vq8,32935
+fmdt/config.py,sha256=ZUXr-CUDih7olqIU0wRAYkEXZNyBiHfNBrpbg61l8o4,7995
+fmdt/core.py,sha256=Bzm6pGjXV6Hko9BeTgz5Ixl_xH8XOukO9Y-o-_DSsfY,15530
+fmdt/db.py,sha256=WUGB2KgLF4WVfopsk0t4ZOJm1K2hsGGWbcWiLxRYolY,40319
+fmdt/download.py,sha256=aVGLn6MvB9FB5ndmHp399II3DCaOrZDm-539xyTltsU,3791
+fmdt/exceptions.py,sha256=33kCV_5SmPNRCCzrSYGcggprO1gA2AhCeCY6lYSH-ec,348
+fmdt/res.py,sha256=dsOBd9TfUDyR1E40OCqM8Bm4l2KA2n968aur6uxB4W4,15158
+fmdt/stats.py,sha256=mrgBeFeFTso_QpP_G2zTQjpmq8czickq7BYHC5kdA2s,1084
+fmdt/tests.py,sha256=tAYIICMqg5LVOWanlfWtHcKgy2uvSRfVOmJl4R3dLLc,3562
+fmdt/truth.py,sha256=X75ADsXeowMl_d_-CZk29k7rGrGsnyL65qwB5sugQOs,19740
+fmdt/utils.py,sha256=sqE-rA78rr_bLqHXTp_HdCMFRNXPGJGbamVYS4GdhtM,10095
+fmdt_python-0.0.42.dist-info/LICENSE,sha256=7oPhtg5rNNa1IVBDV-A7WY-TVhhBIwuaT4dKmd9sqMY,1068
+fmdt_python-0.0.42.dist-info/METADATA,sha256=TvS536WIQzMakDT3N8nMbOxrjRZmb5EUWv_-KV5kl1Q,1611
+fmdt_python-0.0.42.dist-info/WHEEL,sha256=pkctZYzUS4AYVn6dJ-7367OJZivF2e8RA9b_ZBjif18,92
+fmdt_python-0.0.42.dist-info/top_level.txt,sha256=mfR9HxoivulFZbTMXWUH0J2-8270GBpGjQe69ki4V8E,5
+fmdt_python-0.0.42.dist-info/RECORD,,

fmdt_python-0.0.42.dist-info/top_level.txt

+fmdt

fmdt_python-0.0.42.dist-info/WHEEL

+Wheel-Version: 1.0
+Generator: bdist_wheel (0.40.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

fmdt/stats.py

+"""Module dedicated to reporting the statistics of the current database"""
+
+import fmdt.db
+
+from fmdt.download import (
+    get_db_dir
+)
+
+from fmdt.utils import (
+    join
+)
+
+import sqlite3
+import pandas as pd
+
+def num_videos(
+    db_file = "videos.db",
+    db_dir = get_db_dir()
+) -> pd.DataFrame:
+
+    con = sqlite3.connect(join(db_dir, db_file))
+
+    df = pd.read_sql_query("""
+            SELECT type, count(*)
+            FROM video
+            GROUP BY type;
+        """,
+        con
+    )
+
+    df.rename(
+        columns={"count(*)": "n_clips"},
+        inplace=True
+    )
+
+    con.close()
+
+    return df
+
+
+def num_meteors(
+    db_file = "videos.db",
+    db_dir = get_db_dir()
+) -> pd.DataFrame:
+
+    con = sqlite3.connect(join(db_dir, db_file))
+
+    df = pd.read_sql_query("""
+        SELECT type, count(*) 
+        FROM video
+        INNER JOIN human_detections as hd
+        ON video.name = hd.video_name
+        GROUP BY type;
+        """,
+        con
+    )
+
+    df.rename(
+        columns={"count(*)": "n_clips"},
+        inplace=True
+    )
+
+    con.close()
+
+    return df

fmdt/__init__.py

@@ -76,5 +76,16 @@ from fmdt.core import (
     load_all,
-    info as local_info
+    get_video,
+    info as local_info,
+    retrieve_table_video,
+    retrieve_table_video_clips,
+    retrieve_table_best_detections,
+    retrieve_table_detect_args,
+    retrieve_table_human_detections
 )
 
+from fmdt.stats import (
+    num_videos,
+    num_meteors
+)
+
 init_cache()
@@ -88,4 +99,4 @@
 MINOR_VERSION = 0
-PATCH         = 41 
+PATCH         = 42 
 
 VERSION = str(MAJOR_VERSION) + "." + str(MINOR_VERSION) + "." + str(PATCH)

fmdt/args.py

 """Arguments Class to store shared parameters when calling a chain of .detect(args).visu().split()"""
 import fmdt.api
-# import fmdt.res
 import fmdt.core
@@ -14,2 +13,37 @@ import fmdt.utils
 
+# Hardcoded default detect values for FMDT version v1.0.0-21-g7cba20b4 (7cba20b4)
+# These are used to convert between args in our database and args in python.
+# However, they are NOT used as the default arguments to our .detect() API
+# since the authors of FMDT should be able to change the default values of their
+# own program and the api should allow that.
+_DEFAULT_DETECT_ARGS = {
+    "vid_in_start": 0,
+    "vid_in_stop": 0,
+    "vid_in_skip": 0,
+    "vid_in_buff": False,
+    "vid_in_loop": 1,
+    "vid_in_threads": 0,
+    "light_min": 55,
+    "light_max": 80,
+    "ccl_fra_path": None,
+    "ccl_fra_id": False,
+    "mrp_s_min": 3,
+    "mrp_s_max": 1000,
+    "knn_k": 3,
+    "knn_d": 10,
+    "knn_s": 0.125,
+    "trk_ext_d": 10,
+    "trk_ext_o": 3,
+    "trk_angle": 20.0,
+    "trk_star_min": 15,
+    "trk_meteor_min": 3,
+    "trk_meteor_max": 100,
+    "trk_ddev": 4.0,
+    "trk_all": False,
+    "trk_bb_path": None,
+    "trk_mag_path": None,
+    "log_path": None
+}
+
+# Configuration to find fmdt-detect if it doesn't exist on the path
 __EXECUTABLE_PATH = None
@@ -24,7 +58,3 @@
 
-class Result:
-    pass
-    
-
-def filter_dict(d: dict):
+def _filter_dict(d: dict):
     """Filter out None values in a dict, returning a new dict"""
@@ -39,3 +69,3 @@ out = {}
 
-def row_to_dict(row: pd.Series) -> dict:
+def _row_to_dict(row: pd.Series) -> dict:
     """Convert the non Na values of a pd.DataFrame row to a dict"""
@@ -109,7 +139,11 @@ out = {}
         self.trk_mag_path = trk_mag_path
+        self.log_path = log_path
         self.trk_out_path = trk_out_path
-        self.log_path = log_path
 
-    def to_dict(self) -> dict:
-        return {
+    def to_dict(
+            self,
+            subset: list[str] = None
+    ) -> dict:
+
+        d = {
             "vid_in_path": self.vid_in_path, 
@@ -144,2 +178,65 @@ "vid_in_start": self.vid_in_start,
         }
+
+        if subset is None:
+            return d
+        
+        else:
+            dsub = {}
+            for k in subset:
+                dsub[k] = d[k]
+        
+            return dsub
+
+    def to_stripped_dict(self) -> dict:
+        """Return a stripped dictionary that drops all parameters pertaining to paths"""
+        d = self.to_dict()
+
+        del d["vid_in_path"]
+        del d["ccl_fra_path"]
+        del d["trk_bb_path"]
+        del d["trk_mag_path"]
+        del d["log_path"]
+        del d["trk_out_path"]
+
+        return d
+
+    def to_default_stripped_dict(self) -> dict:
+        """Produce a stripped dict where all the none values are set to their default""" 
+
+        d = self.to_stripped_dict()
+
+        for k, v in d.items():
+            if v is None:
+                d[k] = _DEFAULT_DETECT_ARGS[k]
+
+        return d
+
+    def to_sql_insert(
+            self,
+            id: int,
+            table_name: str = "detect_args"
+        ) -> str:
+        """Create the SQL insertion query that adds this detect args to a table named `table_name`"""
+
+        dict_values = self.to_default_stripped_dict().values()
+        n_values = len(dict_values)
+
+        sql = f"INSERT INTO {table_name} VALUES ({id}, "
+
+        for i, v in enumerate(dict_values):
+
+            if i == n_values - 1:
+                break
+                
+            if isinstance(v, bool):
+                v = int(v)
+
+            sql += f"{v}, "
+
+        if isinstance(v, bool):
+            v = int(v)
+
+        sql += f"{v});"
+
+        return sql
     
@@ -200,2 +297,37 @@ def to_reduced_dict(self) -> dict:
 
+    @staticmethod
+    def sql_create_table(table_name: str = "detect_args") -> str:
+        """Return the SQL instructions to create a DetectArgs table named `table_name`"""
+        sql = (
+            f"""
+            CREATE TABLE {table_name} (
+                id_args INTEGER NON NULL PRIMARY KEY,
+                vid_in_start INTEGER,
+                vid_in_stop INTEGER,
+                vid_in_skip INTEGER,
+                vid_in_buff BOOLEAN,
+                vid_in_loop INTEGER,
+                vid_in_threads INTEGER,
+                light_min INTEGER,
+                light_max INTEGER,
+                ccl_fra_id INETEGER,
+                mrp_s_min INTEGER,
+                mrp_s_max INTEGER,
+                knn_k INTEGER,
+                knn_d INTEGER,
+                knn_s NUMERIC,
+                trk_ext_d INTEGER,
+                trk_ext_o INTEGER,
+                trk_angle NUMERIC,
+                trk_star_min INTEGER,
+                trk_meteor_min INTEGER,
+                trk_meteor_max INTEGER,
+                trk_ddev NUMERIC,
+                trk_all INTEGER
+            );
+            """
+        )
+
+        return sql
+
 class VisuArgs:
@@ -363,4 +495,4 @@
                               vid_in_threads=vid_in_threads,
-                              light_min =light_min ,
-                              light_max =light_max ,
+                              light_min =light_min,
+                              light_max =light_max,
                               ccl_fra_path=ccl_fra_path,
@@ -634,3 +766,4 @@ ccl_fra_id=ccl_fra_id,
         log: bool = False,
-        timeout: float = None
+        timeout: float = None,
+        **args # any leftovers, useful when converting sql query to args object
     ) -> Args:
@@ -637,0 +770,0 @@ """Convert the parameters used in fmdt.detect into an Args object"""

fmdt/db.py

@@ -22,2 +22,3 @@ """Module dealing with the database stuff"""
 import os
+from sys import exit
 
@@ -128,2 +129,52 @@ VIDEOS_FILE = fmdt.config.dir() + "/videos.db"
 
+    def detect_best(
+            self,
+            log = False 
+        ) -> fmdt.res.DetectionResult:
+        """Use the arguments from our best_detections database file to call a detection. 
+        
+        If there is no recorded best detection, then use FMDT's default parameters  
+        """
+
+        if self.has_best_detection():
+
+            best_args = self.best_args()
+            best_args.detect_args.vid_in_path = self.full_path()
+            best_args.log = log
+
+            return best_args.detect()
+
+        else:
+
+            fmdt.utils.stderr(f"WARNING: {self} has no records in best_detections table; using default FMDT arguments")
+            return self.detect()
+
+    def validate_best(
+            self,
+            log = False
+        ) -> bool:
+        """Call detect_best().check() and see if the trk_rate is the same as what is recorded in our database
+        
+
+        Return
+        ------
+
+        validated (bool): True when the trk_rate of a fresh run with the best args matches the trk_rate of the
+            corresponding entry in our database
+        """
+
+        if self.has_best_detection():
+
+            best_args, trk_rate, true_pos = self.best_detection()
+            d_args_stripped = best_args.detect_args.to_stripped_dict()
+
+            c_res = self.detect(**d_args_stripped).check()
+
+            return trk_rate == c_res.trk_rate() and true_pos == c_res.true_pos()
+
+        else:
+
+            fmdt.utils.stderr(f"WARNING: {self} has no records in best_detections table; Video.validate_best() returning false")
+            return False
+
     # def visu
@@ -319,3 +370,60 @@ def detect(self,
         return len(self.meteors(db_filename, db_dir)) > 0
+    
+    def has_best_detection(
+            self,
+            db_file: str = "videos.db",
+            db_dir = DEFAULT_DATA_DIR
+        ) -> bool:
+        """Check if this Video has a best detection stored in our database"""
 
+        id = self.id(db_file, db_dir)
+        return query_best_detection(id, False, db_file, db_dir)
+
+    def best_detection(
+            self,
+            db_file: str = "videos.db",
+            db_dir = DEFAULT_DATA_DIR
+        ) -> tuple[fmdt.Args, float, int]:
+
+        id = self.id(db_file, db_dir)
+
+        if self.has_best_detection(db_file, db_dir):
+            args, trk_rate, n_true_pos = retrieve_best_detection(id, db_file=db_file, db_dir=db_dir)
+            args.detect_args.vid_in_path = self.full_path()
+            return args, trk_rate, n_true_pos
+
+        else:
+            fmdt.utils.stderr(f"WARNING: {self} has no best detection in {fmdt.utils.join(db_dir, db_file)}, VideoClip.best_detection(); returning None")
+            return None
+
+    def best_args(
+            self,
+            db_file: str = "videos.db",
+            db_dir = DEFAULT_DATA_DIR
+        ) -> fmdt.Args:
+
+        args, _, _ = self.best_detection(db_file, db_dir)
+        return args
+
+    def best_trk_rate(
+            self,
+            db_file: str = "videos.db",
+            db_dir = DEFAULT_DATA_DIR
+        ) -> float:
+
+        _, trk_rate, _ = self.best_detection(db_file, db_dir)
+
+        return trk_rate
+        
+
+    def best_true_pos(
+            self,
+            db_file: str = "videos.db",
+            db_dir = DEFAULT_DATA_DIR
+        ) -> int:
+
+        _, _, n_true_pos = self.best_detection(db_file, db_dir)
+
+        return n_true_pos
+
     def exists(self) -> bool:
@@ -523,3 +631,3 @@ """Check whether the video path in self.full_path() exists on disk"""
             """Check if the meteors start frame is in the clip"""
-            return hum_det.start_frame >= self.start_frame and hum_det.start_frame <= self.end_frame
+            return hum_det.start_frame >= self.start_frame and hum_det.start_frame < self.end_frame
         
@@ -535,2 +643,32 @@ def modify_meteor(m: fmdt.HumanDetection):
         return [modify_meteor(m) for m in p_meteors if pred(m)]
+
+    def has_best_detection(
+            self,
+            db_file: str = "videos.db",
+            db_dir = DEFAULT_DATA_DIR
+        ) -> bool:
+        """Check if this Video has a best detection stored in our database"""
+
+        id = self.id(db_file, db_dir)
+        return query_best_detection(id, True, db_file, db_dir)
+
+    def best_detection(
+            self,
+            db_file: str = "videos.db",
+            db_dir = DEFAULT_DATA_DIR
+        ) -> tuple[fmdt.Args, float, int] | None:
+        """
+        Get the best detection from our database, retrieved as a (args, trk_rate, n_true_pos) tuple. 
+
+        If no best detection exists, return None.
+        
+        """
+
+        id = self.id(db_file, db_dir)
+
+        if self.has_best_detection(db_file, db_dir):
+            return retrieve_best_detection(id, video_clip=True, db_file=db_file, db_dir=db_dir)
+        else:
+            fmdt.utils.stderr(f"WARNING: {self} has no best detection in {fmdt.utils.join(db_dir, db_file)}, VideoClip.best_detection(); returning None")
+            return None
     
@@ -552,2 +690,68 @@ def parent_path(self) -> str:
 
+    def parent(
+            self,
+            db_file = "videos.db",
+            db_dir = DEFAULT_DATA_DIR
+        ) -> Video:
+        """Retrieve a Video object that this clip was created from"""
+
+        videos = retrieve_videos(db_file, db_dir)
+        par_list = [v for v in videos if v.name == self.name]
+
+        if len(par_list) != 1:
+            raise DatabaseError(f"Video clip {self} did not find parent in database {fmdt.utils.join(db_dir, db_file)}")
+
+        return par_list[0]
+
+
+    def parent_id(
+            self,
+            db_file = "videos.db",
+            db_dir = DEFAULT_DATA_DIR
+        ) -> int:
+        """Retrieve the id of the parent video in our database file"""
+        return self.parent(db_file, db_dir).id()
+
+    # @override
+    def id(
+            self,
+            db_file = "videos.db",
+            db_dir = DEFAULT_DATA_DIR
+        ) -> int:
+        """Retrieve the clip id of this clip in our database using (parent_id, start_frame, end_frame) as a key
+        
+        Return
+        ------
+        clip_id (int): The id of this clip in our database, if it exists. If the database does not contain
+            a clip with the corresponding key, raise DatabaseError
+        
+        """
+
+        con = sqlite3.connect(fmdt.utils.join(db_dir, db_file))
+
+        try:
+
+            df = pd.read_sql_query(f"""
+                                    SELECT clip_id 
+                                    FROM video_clips
+                                    WHERE parent_id = {self.parent_id()}
+                                    AND start_frame = {self.start_frame}
+                                    AND end_frame = {self.end_frame} 
+                                    """, 
+                                    con)
+        except Exception as db_err:
+            
+            print(db_err)
+            print(colored("Potential solution: update your database file with fmdt.download_dbs()", "green"))
+            con.close()
+            exit(1)
+
+        con.close()
+        
+        if len(df) != 1:
+            raise DatabaseError(f"{self} has no matches in database {fmdt.utils.join(db_file, db_dir)}")
+
+        return df.iloc[0,0]
+
+
     @staticmethod
@@ -589,17 +793,10 @@ def from_pd_row(
         require_gt = False,
-        require_exist = False
+        require_exist = False,
+        require_best_det = False
     ) -> list[Video]:
     """Load draco6 `Video` objects that are stored in the `db_dir`/`filename` .db file"""
 
-    vids = retrieve_videos(db_filename, db_dir)
-    d6 = [v for v in vids if v.is_draco6()]
-
-    if require_gt:
-        d6 = [v for v in d6 if v.has_meteors(db_filename, db_dir)]
-
-    if require_exist:
-        d6 = [v for v in d6 if v.exists()]
+    vids = retrieve_videos(db_filename, db_dir, require_gt, require_exist, require_best_det)
+    return [v for v in vids if v.is_draco6()]
     
-    return d6
-
 def load_draco12(
@@ -609,16 +806,10 @@ db_filename: str = "videos.db",
         require_gt = False,
-        require_exist = False
+        require_exist = False,
+        require_best_det = False
     ) -> list[Video]:
 
-    vids = retrieve_videos(db_filename, db_dir)
-    d12 = [v for v in vids if v.is_draco12()]
+    vids = retrieve_videos(db_filename, db_dir, require_gt, require_exist, require_best_det)
+    return [v for v in vids if v.is_draco12()]
 
-    if require_gt:
-        d12 = [v for v in d12 if v.has_meteors(db_filename, db_dir)]
 
-    if require_exist:
-        d12 = [v for v in d12 if v.exists()]
-    
-    return d12
-
 def load_window(
@@ -628,43 +819,18 @@ db_filename: str = "videos.db",
         require_gt = False,
-        require_exist = False
+        require_exist = False,
+        require_best_det = False
     ) -> list[Video]:
 
-    vids = retrieve_videos(db_filename, db_dir)
-    win = [v for v in vids if v.is_window()]
+    vids = retrieve_videos(db_filename, db_dir, require_gt, require_exist, require_best_det)
+    return [v for v in vids if v.is_window()]
 
-    if require_gt:
-        win = [v for v in win if v.has_meteors(db_filename, db_dir)]
-
-    if require_exist:
-        win = [v for v in win if v.exists()]
-    
-    return win
-
 def load_window_clips(
         db_file = "videos.db",
         db_dir = DEFAULT_DATA_DIR,
-        require_exist = False
+        require_exist = False,
+        require_best_det = False
     ) -> list[VideoClip]:
     """Load all of the clips associated with all of our windows objects"""
+    return retrieve_video_clips(db_file, db_dir, require_exist, require_best_det)
 
-    db_path = fmdt.utils.join(db_dir, db_file)
-    con = sqlite3.connect(db_path)
-
-    df: pd.DataFrame = pd.read_sql_query("select * from video_clips", con = con)
-
-    con.close()
-
-    if require_exist:
-
-        out = []
-        for _, row in df.iterrows():
-            v = VideoClip.from_pd_row(row, db_file, db_dir)
-            if v.exists():
-                out.append[v]
-
-        return out 
-
-    else:
-        return [VideoClip.from_pd_row(row, db_file, db_dir) for _, row in df.iterrows()]
-
     
@@ -682,23 +848,43 @@ def load_demo(db_filename: str = "videos.db", db_dir = DEFAULT_DATA_DIR) -> list[Video]:
         db_dir = DEFAULT_DATA_DIR,
-        require_gt = True
-    ) -> list:
-    """Load all videos that have a ground truth in our database"""
-    draco6  = load_draco6 (db_filename, db_dir, require_gt)
-    draco12 = load_draco12(db_filename, db_dir, require_gt)
-    windows = load_window_clips(db_filename, db_dir)
+        require_gt = True,
+        require_exist = True,
+        require_best_det = False
+    ) -> list[Video]:
+    """Load all Draco6, Draco12, and window_clips
+    
+    The default behavior is to load all the videos that have at least one ground truth and exist on disk. We can also 
+    require that a video has a best detection with the require_best_det parameter
 
+    Parameters
+    ----------
+    require_gt (bool): When True, all returned videos have a ground truth in our database
+        default: True
+    require_exist (bool): When True, all returned videos exist on disk
+        default: True 
+    require_best_det (bool): When True, all returned videos have a recorded best detection in the best_detections table
+        of our database.
+        default: False
+
+    """
+
+    draco6  = load_draco6 (db_filename, db_dir, require_gt, require_exist, require_best_det)
+    draco12 = load_draco12(db_filename, db_dir, require_gt, require_exist, require_best_det)
+    windows = load_window_clips(db_filename, db_dir, require_exist, require_best_det)
+
     return draco6 + draco12 + windows
 
-
 def retrieve_videos(
-        db_filename: str = "videos.db",
-        db_dir = DEFAULT_DATA_DIR
+        db_file: str = "videos.db",
+        db_dir = DEFAULT_DATA_DIR,
+        require_gt = False,
+        require_exist = False,
+        require_best_det = False
     ) -> list[Video]:
 
     """Read in the videos stored in 'videos.db' into a list of fmdt.Video"""
-    db_path = fmdt.utils.join(db_dir, db_filename)
+    db_path = fmdt.utils.join(db_dir, db_file)
 
     # Download if the database file requested doesnt exist
     if not os.path.exists(db_path):
-        fmdt.download.download_videos_db(db_filename, log=False, overwrite=False, dir=db_dir)
+        fmdt.download.download_videos_db(db_file, log=False, overwrite=False, dir=db_dir)
 
@@ -709,4 +895,39 @@ con = sqlite3.connect(db_path)
 
-    return [fmdt.Video.from_pd_row(df.iloc[i]) for i in range(len(df))]
+    vids = [fmdt.Video.from_pd_row(df.iloc[i]) for i in range(len(df))]
 
+    if require_gt:
+        vids = [v for v in vids if v.has_meteors(db_file, db_dir)]
+    
+    if require_exist:
+        vids = [v for v in vids if v.exists()]
+
+    if require_best_det:
+        vids = [v for v in vids if v.has_best_detection(db_file, db_dir)]
+
+    return vids
+
+def retrieve_video_clips(
+        db_file: str = "videos.db",
+        db_dir = DEFAULT_DATA_DIR,
+        require_exist = False,
+        require_best_det = False
+    ) -> list[Video]:
+
+    db_path = fmdt.utils.join(db_dir, db_file)
+    con = sqlite3.connect(db_path)
+
+    df: pd.DataFrame = pd.read_sql_query("select * from video_clips", con = con)
+    clips = [VideoClip.from_pd_row(row, db_file, db_dir) for _, row in df.iterrows()]
+
+    con.close()
+
+    if require_exist:
+        clips = [c for c in clips if c.exists()]
+
+    if require_best_det:
+        clips = [c for c in clips if c.has_best_detection()]
+
+    return clips
+
+
 def retrieve_meteors(
@@ -734,3 +955,141 @@ video_name: str,
 
+def query_best_detection(
+        id: int,
+        video_clip: bool = False,
+        db_file: str = "videos.db",
+        db_dir = DEFAULT_DATA_DIR
+    ) -> bool:
 
+    """Test if there is a best detection associated with the provided id"""
+
+    db_full_path = fmdt.utils.join(db_dir, db_file)
+    con = sqlite3.connect(db_full_path)
+
+    if video_clip:
+
+        df = pd.read_sql_query(f"""
+            SELECT * FROM best_detections
+            WHERE id_video_clip = {id};
+            """, con)
+    
+    else:
+
+        df = pd.read_sql_query(f"""
+            SELECT * FROM best_detections
+            WHERE id_video = {id};
+            """, con)
+
+    return len(df) == 1
+
+
+def retrieve_best_detection_df(
+        id: int,
+        video_clip: bool = False,
+        db_file: str = "videos.db",
+        db_dir = DEFAULT_DATA_DIR
+    ) -> pd.DataFrame:
+
+    db_filename = fmdt.utils.join(db_dir, db_file)
+
+    con = sqlite3.connect(db_filename)
+
+    _ID_ARGS_COL = 2
+
+    if video_clip:
+        df = pd.read_sql_query(f"""
+            SELECT * FROM best_detections 
+            INNER JOIN detect_args
+            ON best_detections.id_args = detect_args.id_args
+            WHERE id_video_clip = {id};
+        """, con)
+        
+        df = df.drop(df.columns[_ID_ARGS_COL], axis=1)
+
+    else:
+        df = pd.read_sql_query(f"""
+            SELECT * FROM best_detections 
+            INNER JOIN detect_args
+            ON best_detections.id_args = detect_args.id_args
+            WHERE id_video = {id};
+        """, con)
+        
+        df = df.drop(df.columns[_ID_ARGS_COL], axis=1)
+
+    con.close()
+    # Now we want to convert this df into an Args object.
+    if len(df) != 1:
+
+        if video_clip:
+            raise DatabaseError(f"Error accessing best args for VideoClip by id {id} from {db_filename}")
+        else:
+            raise DatabaseError(f"Error accessing best args for Video by id {id} from {db_filename}")
+    
+
+    return df
+
+def retrieve_best_arg(
+        id: int,
+        video_clip: bool = False,
+        db_file: str = "videos.db",
+        db_dir = DEFAULT_DATA_DIR
+    ) -> fmdt.Args:
+    """
+    Retrieve the Args object that is associated with the passed Video or VideoClip id
+    
+    Parameters
+    ----------
+    id (int): unique identifier of the Video or VideoClip that we would like to retrieve.
+        when video_clip is True, search for a match in the id_video_clip column of our
+        best_detection table
+    video_clip (bool): Informs this function if we want to retrieve the best args for a VideoClip (True) or 
+        for a Video (false)
+
+    Examples
+    --------
+    >>> args = fmdt.retrieve_best_args(1, video_clip=False) 
+
+    Returns the best Args associated with the *Video* whose id is 1 (Draconids-6mm1.05-0750-164200.avi)
+
+    >>> args = fmdt.retrieve_best_args(4, video_clip=True)
+
+    Returns the vest Args associated with the **VideoClip** whose id is 4  (window_3_sony_0400-0405UTC.mp4 [2823, 2862])
+
+    """
+    df = retrieve_best_detection_df(id, video_clip, db_file, db_dir)
+
+    dict = df.to_dict("records")[0]
+    return fmdt.detect_args(**dict)
+
+def retrieve_best_detection(
+        id: int,
+        video_clip: bool = False,
+        db_file: str = "videos.db",
+        db_dir = DEFAULT_DATA_DIR
+    ) -> tuple[fmdt.Args, float, int]:
+    """
+    Retrieve the best detection result (args, trk_rate, true_pos) associated with a Video or VideoClip id
+    
+    Parameters
+    ----------
+    id (int): unique identifier of the Video or VideoClip that we would like to retrieve.
+        when video_clip is True, search for a match in the id_video_clip column of our
+        best_detection table
+    video_clip (bool): Informs this function if we want to retrieve the best args for a VideoClip (True) or 
+        for a Video (false)
+
+    Return
+    ------
+
+    best_detection (tuple): A tuple of the form (args, trk_rate, true_pos)
+
+    """
+    df = retrieve_best_detection_df(id, video_clip, db_file, db_dir)
+
+    dict = df.to_dict("records")[0]
+    args = fmdt.detect_args(**dict)
+    trk_rate = df["trk_rate"].iloc[0]
+    n_true_pos = df["true_pos"].iloc[0]
+
+    return args, trk_rate, n_true_pos
+    
 def get_video_diagnostics(vids: list[Video]) -> tuple[int, int]:
@@ -790,2 +1149,9 @@ """Print information about the local environment"""
 
+def get_video(selector: str | int) -> Video | None:
+    if isinstance(selector, str):
+        return get_video_by_name(selector)
+    elif isinstance(selector, int):
+        return get_video_by_id(selector)
+    else:
+        raise TypeError(f"get_video can only access videos with an `int` or `str`. Passed object type: {type(selector)}")
 
@@ -800,2 +1166,12 @@ def get_video_by_id(id: int) -> Video | None:
 
+
+
+def get_video_by_name(name: str) -> Video | None:
+    videos = retrieve_videos()
+    v = [v for v in videos if v.name == name]
+    if len(v) != 0:
+        return v[0]
+    else:
+        return None
+
 def get_video_by_ids(ids: list[int]) -> Video | None:
@@ -810,1 +1186,91 @@ return [get_video_by_id(id) for id in ids if not get_video_by_id(id) is None]
 
+import json
+# ==================== Functions dealing with Json output =================== #
+
+def best_draco6(
+        parameter_subset: list[str] = ["light_min", "light_max", "kdd_n"],
+        db_file = "videos.db",
+        db_dir = DEFAULT_DATA_DIR
+) -> str:
+
+    # get the draco 6 videos along with their arguments
+    # db_filename = fmdt.utils.join(db_dir, db_file)
+
+    # con = sqlite3.connect(db_filename)
+    
+    # df = pd.read_sql_query(f"""
+    #     SELECT * FROM best_detections as bd
+    #     INNER JOIN video as v
+    #     ON bd.id_video = v.id
+    #     INNER JOIN detect_args as da
+    #     ON bd.id_args = da.id_args
+    #     WHERE v.type = 'DRACO6'
+    #     """,
+    #     con)
+
+    # print(df)
+
+    # con.close()
+
+    d6 = load_draco6(require_gt = True, require_exist = True, require_best_det = True)
+
+    best_dets = [(d.name, d.best_detection()) for d in d6]
+
+
+
+
+
+
+
+    
+
+
+
+# ======================= Load Relational Database tables as pd.DataFrames ==================== #
+
+def retrieve_table(
+    table_name: str,
+    db_file = "videos.db",
+    db_dir  = DEFAULT_DATA_DIR
+) -> pd.DataFrame:
+
+    db_full_path = fmdt.utils.join(db_dir, db_file)
+
+    con = sqlite3.connect(db_full_path)
+
+    sql = f"""select * from {table_name}"""
+    df = pd.read_sql_query(sql, con)
+
+    con.close()
+
+    return df
+
+def retrieve_table_video(
+    db_file = "videos.db",
+    db_dir  = DEFAULT_DATA_DIR
+) -> pd.DataFrame:
+    return retrieve_table("video", db_file, db_dir)
+
+def retrieve_table_video_clips(
+    db_file = "videos.db",
+    db_dir  = DEFAULT_DATA_DIR
+) -> pd.DataFrame:
+    return retrieve_table("video_clips", db_file, db_dir)
+
+def retrieve_table_best_detections(
+    db_file = "videos.db",
+    db_dir  = DEFAULT_DATA_DIR
+) -> pd.DataFrame:
+    return retrieve_table("best_detections", db_file, db_dir)
+
+def retrieve_table_human_detections(
+    db_file = "videos.db",
+    db_dir  = DEFAULT_DATA_DIR
+) -> pd.DataFrame:
+    return retrieve_table("human_detections", db_file, db_dir)
+
+def retrieve_table_detect_args(
+    db_file = "videos.db",
+    db_dir  = DEFAULT_DATA_DIR
+) -> pd.DataFrame:
+    return retrieve_table("detect_args", db_file, db_dir)

fmdt/exceptions.py

@@ -7,2 +7,6 @@ class GroundTruthError(Exception):
     """Raised when we have issues with the `log_path` parameter"""
+    pass
+
+class DatabaseError(Exception):
+    """Raised when there is an issue retrieving information from our database"""
     pass

fmdt/res.py

@@ -462,2 +462,14 @@ """Store the results of a run to fmdt-detect"""
         return self.meteor_stats()["trk_rate"]
+
+    def true_pos(self) -> int:
+        return self.meteor_stats()["tpos"]
+
+    def true_neg(self) -> int:
+        return self.meteor_stats()["tneg"]
+
+    def false_pos(self) -> int:
+        return self.meteor_stats()["fpos"]
+
+    def false_neg(self) -> int:
+        return self.meteor_stats()["fneg"]
     
@@ -464,0 +476,0 @@ def gts(self) -> int:

fmdt_python-0.0.41.dist-info/LICENSE

-MIT License
-
-Copyright (c) 2023 Evan Voyles
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.

fmdt_python-0.0.41.dist-info/METADATA

-Metadata-Version: 2.1
-Name: fmdt-python
-Version: 0.0.41
-Summary: utlity functions for fast meteor detection toolbox
-Author-email: Evan Voyles <[e-mail]>
-Project-URL: fmdt, https://github.com/alsoc/fmdt
-Project-URL: Homepage, https://github.com/ejovo13/fmdt_scripts
-Classifier: Programming Language :: Python :: 3
-Classifier: License :: OSI Approved :: MIT License
-Classifier: Operating System :: OS Independent
-Requires-Python: >=3.10
-Description-Content-Type: text/markdown
-License-File: LICENSE
-Requires-Dist: numpy
-Requires-Dist: ffmpeg-python
-Requires-Dist: requests
-Requires-Dist: termcolor
-Requires-Dist: matplotlib
-Requires-Dist: pandas
-Requires-Dist: jsonpickle
-Requires-Dist: appdirs
-Requires-Dist: Deprecated
-
-A series of Python scripts to facilitate the processing of [fmdt's](https://github.com/alsoc/fmdt) output
-
-Scripts for video editing rely on [ffmpeg-python's](https://github.com/kkroening/ffmpeg-python) simple Python bindings to ffmpeg. Make sure you install `ffmpeg-python` before trying any of the video editing functionality.
-
-### Installation
-
-```
-pip install fmdt-python
-```
-
-`fmdt/core.py` should contain the functions that are called directly in scripts.
-`fmdt/utils.py` contains utility functions that `fmdt.core` makes use of.
-
-Example to split a video using tracking information already provided by `fmdt-detect`:
-
-```
-import fmdt
-
-fmdt.split_video_at_meteors("demo.mp4", "ex1_detect_tracks.txt")
-```
-
-#### TODO
-
-- [x] Upload fmdt to pip so that we can download fmdt and call scripts from anywhere
-- [x] Add API to call fmdt executables like `fmdt-detect` and `fmdt-visu`

fmdt_python-0.0.41.dist-info/RECORD

-fmdt/__init__.py,sha256=c2_mqwk3mNmc6gCjhc86j8PnaLGLRhHH7O9GoYWLF5A,1386
-fmdt/__test__.py,sha256=FXbpPO1x5EatXgegxcJLGb1iUpWMCz9xuSvOWc0zr8A,1919
-fmdt/api.py,sha256=rFHKXzmOnQnoXUJC13QjbXYXfwFZdmVZ1NnQzX52cH8,12273
-fmdt/args.py,sha256=p8alJhYCrPrDI23bCq1Nge9qDx09vJZurYwVI6I4mDw,29126
-fmdt/config.py,sha256=ZUXr-CUDih7olqIU0wRAYkEXZNyBiHfNBrpbg61l8o4,7995
-fmdt/core.py,sha256=Bzm6pGjXV6Hko9BeTgz5Ixl_xH8XOukO9Y-o-_DSsfY,15530
-fmdt/db.py,sha256=qv-rS4Of0nE8iWcTxttfW6wBS47pRWu9LVbAnOX64uQ,25847
-fmdt/download.py,sha256=aVGLn6MvB9FB5ndmHp399II3DCaOrZDm-539xyTltsU,3791
-fmdt/exceptions.py,sha256=shvijE3euAAFiFO5yehddsiL0DSQBPU4Y3V887CYQzY,225
-fmdt/res.py,sha256=Y0RvFRdikLYGXPFcrhgjf6ibmsk4GkQun13XPxDZZiw,14856
-fmdt/tests.py,sha256=tAYIICMqg5LVOWanlfWtHcKgy2uvSRfVOmJl4R3dLLc,3562
-fmdt/truth.py,sha256=X75ADsXeowMl_d_-CZk29k7rGrGsnyL65qwB5sugQOs,19740
-fmdt/utils.py,sha256=sqE-rA78rr_bLqHXTp_HdCMFRNXPGJGbamVYS4GdhtM,10095
-fmdt_python-0.0.41.dist-info/LICENSE,sha256=7oPhtg5rNNa1IVBDV-A7WY-TVhhBIwuaT4dKmd9sqMY,1068
-fmdt_python-0.0.41.dist-info/METADATA,sha256=ggGe1CXvXNCBjXqFZFt_tnooUMBiCP3MzKvQsuMpnik,1611
-fmdt_python-0.0.41.dist-info/WHEEL,sha256=pkctZYzUS4AYVn6dJ-7367OJZivF2e8RA9b_ZBjif18,92
-fmdt_python-0.0.41.dist-info/top_level.txt,sha256=mfR9HxoivulFZbTMXWUH0J2-8270GBpGjQe69ki4V8E,5
-fmdt_python-0.0.41.dist-info/RECORD,,

fmdt_python-0.0.41.dist-info/top_level.txt

-fmdt

fmdt_python-0.0.41.dist-info/WHEEL

-Wheel-Version: 1.0
-Generator: bdist_wheel (0.40.0)
-Root-Is-Purelib: true
-Tag: py3-none-any
-

Alert delta unavailable

Currently unable to show alert delta for PyPI packages.

Improved metrics

Total package byte prevSize

170551

13.36%

Number of package files

19

5.56%

Lines of code

3999

14.95%