Add support for Finger Print, NFC Card Scan, and Audio Detections

Also refactored code that checks if an event should be backed up into one common shared function.

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,26 @@
+---
+name: Bug report
+about: Create a report to help UPB improve
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+* Unifi Protect Backup version:
+* Unifi Protect version:
+* Python version:
+* Operating System:
+* Are you using a docker container or native?:
+
+### Description
+
+Describe what you were trying to get done.
+Tell us what happened, what went wrong, and what you expected to happen.
+
+### What I Did
+
+```
+Paste the command(s) you ran and the output.
+If there was a crash, please include the traceback here.
+```

README.md

@@ -90,7 +90,6 @@ docker run \
   -e UFP_ADDRESS='UNIFI_PROTECT_IP' \
   -e UFP_SSL_VERIFY='false' \
   -e RCLONE_DESTINATION='my_remote:/unifi_protect_backup' \
-  -v '/path/to/save/clips':'/data' \
   -v '/path/to/rclone.conf':'/config/rclone/rclone.conf' \
   -v '/path/to/save/database':/config/database/ \
   ghcr.io/ep1cman/unifi-protect-backup

unifi_protect_backup/cli.py

@@ -7,13 +7,15 @@ import click
 from aiorun import run  # type: ignore
 from dateutil.relativedelta import relativedelta
 
-from uiprotect.data.types import SmartDetectObjectType
+from uiprotect.data.types import SmartDetectObjectType, SmartDetectAudioType
 
 from unifi_protect_backup import __version__
 from unifi_protect_backup.unifi_protect_backup_core import UnifiProtectBackup
 from unifi_protect_backup.utils import human_readable_to_float
 
-DETECTION_TYPES = ["motion", "ring", "line"] + SmartDetectObjectType.values()
+DETECTION_TYPES = ["motion", "ring", "line", "fingerprint", "nfc"]
+DETECTION_TYPES += [t for t in SmartDetectObjectType.values() if t not in SmartDetectAudioType.values()]
+DETECTION_TYPES += [f"{t}" for t in SmartDetectAudioType.values()]
 
 
 def _parse_detection_types(ctx, param, value):
@@ -251,7 +253,7 @@ a lot of failed downloads with the default downloader.
     "--storage-quota",
     envvar="STORAGE_QUOTA",
     help='The maximum amount of storage to use for storing clips (you can use suffixes like "B", "KiB", "MiB", "GiB")',
-    callback=lambda ctx, param, value: int(human_readable_to_float(value)),
+    callback=lambda ctx, param, value: int(human_readable_to_float(value)) if value is not None else None,
 )
 def main(**kwargs):
     """Python based tool for backing up Unifi Protect event clips as they occur."""

unifi_protect_backup/downloader.py

@@ -114,7 +114,7 @@ class VideoDownloader:
                 output_queue_max_size = human_readable_size(self.upload_queue.maxsize)
                 self.logger.debug(f"Video Download Buffer: {output_queue_current_size}/{output_queue_max_size}")
                 self.logger.debug(f"  Camera: {await get_camera_name(self._protect, event.camera_id)}")
-                if event.type == EventType.SMART_DETECT:
+                if event.type in [EventType.SMART_DETECT, EventType.SMART_AUDIO_DETECT]:
                     self.logger.debug(f"  Type: {event.type.value} ({', '.join(event.smart_detect_types)})")
                 else:
                     self.logger.debug(f"  Type: {event.type.value}")

unifi_protect_backup/downloader_experimental.py

@@ -114,7 +114,7 @@ class VideoDownloaderExperimental:
                 output_queue_max_size = human_readable_size(self.upload_queue.maxsize)
                 self.logger.debug(f"Video Download Buffer: {output_queue_current_size}/{output_queue_max_size}")
                 self.logger.debug(f"  Camera: {await get_camera_name(self._protect, event.camera_id)}")
-                if event.type == EventType.SMART_DETECT:
+                if event.type in [EventType.SMART_DETECT, EventType.SMART_AUDIO_DETECT]:
                     self.logger.debug(f"  Type: {event.type.value} ({', '.join(event.smart_detect_types)})")
                 else:
                     self.logger.debug(f"  Type: {event.type.value}")

unifi_protect_backup/event_listener.py

@@ -3,14 +3,15 @@
 import asyncio
 import logging
 from time import sleep
-from typing import List
+from typing import Set
 
 from uiprotect.api import ProtectApiClient
 from uiprotect.websocket import WebsocketState
 from uiprotect.data.nvr import Event
-from uiprotect.data.types import EventType
 from uiprotect.data.websocket import WSAction, WSSubscriptionMessage
 
+from unifi_protect_backup.utils import wanted_event_type
+
 logger = logging.getLogger(__name__)
 
 
@@ -21,27 +22,27 @@ class EventListener:
         self,
         event_queue: asyncio.Queue,
         protect: ProtectApiClient,
-        detection_types: List[str],
-        ignore_cameras: List[str],
-        cameras: List[str],
+        detection_types: Set[str],
+        ignore_cameras: Set[str],
+        cameras: Set[str],
     ):
         """Init.
 
         Args:
             event_queue (asyncio.Queue): Queue to place events to backup on
             protect (ProtectApiClient): UniFI Protect API client to use
-            detection_types (List[str]): Desired Event detection types to look for
-            ignore_cameras (List[str]): Cameras IDs to ignore events from
-            cameras (List[str]): Cameras IDs to ONLY include events from
+            detection_types (Set[str]): Desired Event detection types to look for
+            ignore_cameras (Set[str]): Cameras IDs to ignore events from
+            cameras (Set[str]): Cameras IDs to ONLY include events from
 
         """
         self._event_queue: asyncio.Queue = event_queue
         self._protect: ProtectApiClient = protect
         self._unsub = None
         self._unsub_websocketstate = None
-        self.detection_types: List[str] = detection_types
-        self.ignore_cameras: List[str] = ignore_cameras
-        self.cameras: List[str] = cameras
+        self.detection_types: Set[str] = detection_types
+        self.ignore_cameras: Set[str] = ignore_cameras
+        self.cameras: Set[str] = cameras
 
     async def start(self):
         """Run main Loop."""
@@ -63,35 +64,10 @@ class EventListener:
         assert isinstance(msg.new_obj, Event)
         if msg.action != WSAction.UPDATE:
             return
-        if msg.new_obj.camera_id in self.ignore_cameras:
-            return
-        if self.cameras and msg.new_obj.camera_id not in self.cameras:
-            return
         if "end" not in msg.changed_data:
             return
-        if msg.new_obj.type not in [
-            EventType.MOTION,
-            EventType.SMART_DETECT,
-            EventType.RING,
-            EventType.SMART_DETECT_LINE,
-        ]:
+        if not wanted_event_type(msg.new_obj, self.detection_types, self.cameras, self.ignore_cameras):
             return
-        if msg.new_obj.type is EventType.MOTION and "motion" not in self.detection_types:
-            logger.extra_debug(f"Skipping unwanted motion detection event: {msg.new_obj.id}")  # type: ignore
-            return
-        if msg.new_obj.type is EventType.RING and "ring" not in self.detection_types:
-            logger.extra_debug(f"Skipping unwanted ring event: {msg.new_obj.id}")  # type: ignore
-            return
-        if msg.new_obj.type is EventType.SMART_DETECT_LINE and "line" not in self.detection_types:
-            logger.extra_debug(f"Skipping unwanted line event: {msg.new_obj.id}")  # type: ignore
-            return
-        elif msg.new_obj.type is EventType.SMART_DETECT:
-            for event_smart_detection_type in msg.new_obj.smart_detect_types:
-                if event_smart_detection_type not in self.detection_types:
-                    logger.extra_debug(  # type: ignore
-                        f"Skipping unwanted {event_smart_detection_type} detection event: {msg.new_obj.id}"
-                    )
-                    return
 
         # TODO: Will this even work? I think it will block the async loop
         while self._event_queue.full():

unifi_protect_backup/missing_event_checker.py

@@ -3,7 +3,7 @@
 import asyncio
 import logging
 from datetime import datetime
-from typing import AsyncIterator, List
+from typing import AsyncIterator, List, Set
 
 import aiosqlite
 from dateutil.relativedelta import relativedelta
@@ -12,6 +12,7 @@ from uiprotect.data.nvr import Event
 from uiprotect.data.types import EventType
 
 from unifi_protect_backup import VideoDownloader, VideoUploader
+from unifi_protect_backup.utils import EVENT_TYPES_MAP, wanted_event_type
 
 logger = logging.getLogger(__name__)
 
@@ -27,9 +28,9 @@ class MissingEventChecker:
         downloader: VideoDownloader,
         uploaders: List[VideoUploader],
         retention: relativedelta,
-        detection_types: List[str],
-        ignore_cameras: List[str],
-        cameras: List[str],
+        detection_types: Set[str],
+        ignore_cameras: Set[str],
+        cameras: Set[str],
         interval: int = 60 * 5,
     ) -> None:
         """Init.
@@ -41,9 +42,9 @@ class MissingEventChecker:
             downloader (VideoDownloader): Downloader to check for on-going downloads
             uploaders (List[VideoUploader]): Uploaders to check for on-going uploads
             retention (relativedelta): Retention period to limit search window
-            detection_types (List[str]): Detection types wanted to limit search
-            ignore_cameras (List[str]): Ignored camera IDs to limit search
-            cameras (List[str]): Included (ONLY) camera IDs to limit search
+            detection_types (Set[str]): Detection types wanted to limit search
+            ignore_cameras (Set[str]): Ignored camera IDs to limit search
+            cameras (Set[str]): Included (ONLY) camera IDs to limit search
             interval (int): How frequently, in seconds, to check for missing events,
 
         """
@@ -53,9 +54,9 @@ class MissingEventChecker:
         self._downloader: VideoDownloader = downloader
         self._uploaders: List[VideoUploader] = uploaders
         self.retention: relativedelta = retention
-        self.detection_types: List[str] = detection_types
-        self.ignore_cameras: List[str] = ignore_cameras
-        self.cameras: List[str] = cameras
+        self.detection_types: Set[str] = detection_types
+        self.ignore_cameras: Set[str] = ignore_cameras
+        self.cameras: Set[str] = cameras
         self.interval: int = interval
 
     async def _get_missing_events(self) -> AsyncIterator[Event]:
@@ -69,12 +70,7 @@ class MissingEventChecker:
             events_chunk = await self._protect.get_events(
                 start=start_time,
                 end=end_time,
-                types=[
-                    EventType.MOTION,
-                    EventType.SMART_DETECT,
-                    EventType.RING,
-                    EventType.SMART_DETECT_LINE,
-                ],
+                types=list(EVENT_TYPES_MAP.keys()),
                 limit=chunk_size,
             )
 
@@ -109,35 +105,23 @@ class MissingEventChecker:
                 if current_upload is not None:
                     uploading_event_ids.add(current_upload.id)
 
-            missing_event_ids = set(unifi_events.keys()) - (db_event_ids | downloading_event_ids | uploading_event_ids)
+            missing_events = {
+                event_id: event
+                for event_id, event in unifi_events.items()
+                if event_id not in (db_event_ids | downloading_event_ids | uploading_event_ids)
+            }
 
             # Exclude events of unwanted types
-            def wanted_event_type(event_id, unifi_events=unifi_events):
-                event = unifi_events[event_id]
-                if event.start is None or event.end is None:
-                    return False  # This event is still on-going
-                if event.camera_id in self.ignore_cameras:
-                    return False
-                if self.cameras and event.camera_id not in self.cameras:
-                    return False
-                if event.type is EventType.MOTION and "motion" not in self.detection_types:
-                    return False
-                if event.type is EventType.RING and "ring" not in self.detection_types:
-                    return False
-                if event.type is EventType.SMART_DETECT_LINE and "line" not in self.detection_types:
-                    return False
-                elif event.type is EventType.SMART_DETECT:
-                    for event_smart_detection_type in event.smart_detect_types:
-                        if event_smart_detection_type not in self.detection_types:
-                            return False
-                return True
-
-            wanted_event_ids = set(filter(wanted_event_type, missing_event_ids))
+            wanted_events = {
+                event_id: event
+                for event_id, event in missing_events.items()
+                if wanted_event_type(event, self.detection_types, self.cameras, self.ignore_cameras)
+            }
 
             # Yeild events one by one to allow the async loop to start other task while
             # waiting on the full list of events
-            for id in wanted_event_ids:
-                yield unifi_events[id]
+            for event in wanted_events.values():
+                yield event
 
             # Last chunk was in-complete, we can stop now
             if len(events_chunk) < chunk_size:

unifi_protect_backup/unifi_protect_backup_core.py

@@ -185,11 +185,11 @@ class UnifiProtectBackup:
             verify_ssl=self.verify_ssl,
             subscribed_models={ModelType.EVENT},
         )
-        self.ignore_cameras = ignore_cameras
-        self.cameras = cameras
+        self.ignore_cameras = set(ignore_cameras)
+        self.cameras = set(cameras)
         self._download_queue: asyncio.Queue = asyncio.Queue()
         self._unsub: Callable[[], None]
-        self.detection_types = detection_types
+        self.detection_types = set(detection_types)
         self._has_ffprobe = False
         self._sqlite_path = sqlite_path
         self._db = None
@@ -283,9 +283,6 @@ class UnifiProtectBackup:
             )
             tasks.append(downloader.start())
 
-            # A way for uploaders to signal they have uploaded a file
-            upload_signal = asyncio.Event()
-
             # Create upload tasks
             #   This will upload the videos in the downloader's buffer to the rclone remotes and log it in the database
             uploaders = []
@@ -298,7 +295,6 @@ class UnifiProtectBackup:
                     self.file_structure_format,
                     self._db,
                     self.color_logging,
-                    upload_signal,
                 )
                 uploaders.append(uploader)
                 tasks.append(uploader.start())
@@ -324,7 +320,11 @@ class UnifiProtectBackup:
 
             if self._storage_quota is not None:
                 storage_quota_purger = StorageQuotaPurge(
-                    self._db, self._storage_quota, upload_signal, self.rclone_destination, self.rclone_purge_args
+                    self._db,
+                    self._storage_quota,
+                    uploader.upload_signal,
+                    self.rclone_destination,
+                    self.rclone_purge_args,
                 )
                 tasks.append(storage_quota_purger.start())
 

unifi_protect_backup/uploader.py

@@ -35,7 +35,6 @@ class VideoUploader:
         file_structure_format: str,
         db: aiosqlite.Connection,
         color_logging: bool,
-        upload_signal: asyncio.Event,
     ):
         """Init.
 
@@ -47,7 +46,6 @@ class VideoUploader:
             file_structure_format (str): format string for how to structure the uploaded files
             db (aiosqlite.Connection): Async SQlite database connection
             color_logging (bool):  Whether or not to add color to logging output
-            upload_signal (asyncio.Event): Set by the uploader to signal an upload has occured
 
         """
         self._protect: ProtectApiClient = protect
@@ -57,7 +55,7 @@ class VideoUploader:
         self._file_structure_format: str = file_structure_format
         self._db: aiosqlite.Connection = db
         self.current_event = None
-        self._upload_signal = upload_signal
+        self._upload_signal = asyncio.Event()
 
         self.base_logger = logging.getLogger(__name__)
         setup_event_logger(self.base_logger, color_logging)

unifi_protect_backup/utils.py

@@ -4,12 +4,13 @@ import asyncio
 import logging
 import re
 from datetime import datetime
-from typing import Optional
+from typing import Optional, Set
 
 from apprise import NotifyType
 from async_lru import alru_cache
 from uiprotect import ProtectApiClient
 from uiprotect.data.nvr import Event
+from uiprotect.data.types import EventType, SmartDetectObjectType, SmartDetectAudioType
 
 from unifi_protect_backup import notifications
 
@@ -454,3 +455,38 @@ async def wait_until(dt):
     """Sleep until the specified datetime."""
     now = datetime.now()
     await asyncio.sleep((dt - now).total_seconds())
+
+
+EVENT_TYPES_MAP = {
+    EventType.MOTION: {"motion"},
+    EventType.RING: {"ring"},
+    EventType.SMART_DETECT_LINE: {"line"},
+    EventType.FINGERPRINT_IDENTIFIED: {"fingerprint"},
+    EventType.NFC_CARD_SCANNED: {"nfc"},
+    EventType.SMART_DETECT: {t for t in SmartDetectObjectType.values() if t not in SmartDetectAudioType.values()},
+    EventType.SMART_AUDIO_DETECT: {f"{t}" for t in SmartDetectAudioType.values()},
+}
+
+
+def wanted_event_type(event, wanted_detection_types: Set[str], cameras: Set[str], ignore_cameras: Set[str]):
+    """Return True if this event is one we want."""
+    if event.start is None or event.end is None:
+        return False  # This event is still on-going
+
+    if event.camera_id in ignore_cameras:
+        return False
+
+    if cameras and event.camera_id not in cameras:
+        return False
+
+    if event.type not in EVENT_TYPES_MAP:
+        return False
+
+    if event.type in [EventType.SMART_DETECT, EventType.SMART_AUDIO_DETECT]:
+        detection_types = set(event.smart_detect_types)
+    else:
+        detection_types = EVENT_TYPES_MAP[event.type]
+    if not detection_types & wanted_detection_types:  # No intersection
+        return False
+
+    return True