Add ability to skip missing events at launch

README.md

@@ -183,6 +183,8 @@ Options:
                                   
                                   More details about supported platforms can be found here:
                                   https://github.com/caronc/apprise
+  --skip-missing                  If set, events which are 'missing' at the start will be ignored. 
+                                  Subsequent missing events will be downloaded (e.g. a missed event)  [default: False]
   --help                          Show this message and exit.
 ```
 
@@ -204,6 +206,7 @@ always take priority over environment variables):
 - `COLOR_LOGGING`
 - `PURGE_INTERVAL`
 - `APPRISE_NOTIFIERS`
+- `SKIP_MISSING`
 
 ## File path formatting
 
@@ -222,6 +225,12 @@ The following fields are provided to the format string:
 
 You can optionally format the `event.start`/`event.end` timestamps as per the [`strftime` format](https://docs.python.org/3/library/datetime.html#strftime-strptime-behavior) by appending it after a `:` e.g to get just the date without the time: `{event.start:%Y-%m-%d}`
 
+## Skipping initially missing events
+If you prefer to avoid backing up the entire backlog of events, and would instead prefer to back up events that occur from 
+now on, you can use the `--skip-missing` flag. This does not enable the periodic check for missing event (e.g. one that was missed by a disconnection) but instead marks all missing events at start-up as backed up.
+
+If you use this feature it is advised that your run the tool once with this flag, then stop it once the database has been created and the events are ignored. Keeping this flag set permanently could cause events to be missed if the tool crashes and is restarted etc.
+
 # A note about `rclone` backends and disk wear
 This tool attempts to not write the downloaded files to disk to minimise disk wear, and instead streams them directly to 
 rclone. Sadly, not all storage backends supported by `rclone` allow "Stream Uploads". Please refer to the `StreamUpload` column on this table to see which one do and don't: https://rclone.org/overview/#optional-features

unifi_protect_backup/cli.py

@@ -151,6 +151,17 @@ If no tags are specified, it defaults to ERROR
 
 More details about supported platforms can be found here: https://github.com/caronc/apprise""",
 )
+@click.option(
+    '--skip-missing',
+    default=False,
+    show_default=True,
+    is_flag=True,
+    envvar='SKIP_MISSING',
+    help="""\b
+If set, events which are 'missing' at the start will be ignored.
+Subsequent missing events will be downloaded (e.g. a missed event)
+""",
+)
 def main(**kwargs):
     """A Python based tool for backing up Unifi Protect event clips as they occur."""
     event_listener = UnifiProtectBackup(**kwargs)

unifi_protect_backup/missing_event_checker.py

@@ -9,6 +9,7 @@ import aiosqlite
 from dateutil.relativedelta import relativedelta
 from pyunifiprotect import ProtectApiClient
 from pyunifiprotect.data.types import EventType
+from pyunifiprotect.data.nvr import Event
 
 from unifi_protect_backup import VideoDownloader, VideoUploader
 
@@ -53,12 +54,7 @@ class MissingEventChecker:
         self.ignore_cameras: List[str] = ignore_cameras
         self.interval: int = interval
 
-    async def start(self):
-        """Main loop."""
-        logger.info("Starting Missing Event Checker")
-        while True:
-            try:
-                logger.extra_debug("Running check for missing events...")
+    async def _get_missing_events(self) -> List[Event]:
         # Get list of events that need to be backed up from unifi protect
         unifi_events = await self._protect.get_events(
             start=datetime.now() - self.retention,
@@ -75,20 +71,17 @@ class MissingEventChecker:
             db_event_ids = {row[0] for row in rows}
 
         # Prevent re-adding events currently in the download/upload queue
-                downloading_event_ids = {event.id for event in self._downloader.download_queue._queue}
+        downloading_event_ids = {event.id for event in self._downloader.download_queue._queue}  # type: ignore
         current_download = self._downloader.current_event
         if current_download is not None:
             downloading_event_ids.add(current_download.id)
 
-                uploading_event_ids = {event.id for event, video in self._uploader.upload_queue._queue}
+        uploading_event_ids = {event.id for event, video in self._uploader.upload_queue._queue}  # type: ignore
         current_upload = self._uploader.current_event
         if current_upload is not None:
             uploading_event_ids.add(current_upload.id)
 
-                missing_event_ids = set(unifi_events.keys()) - (
-                    db_event_ids | downloading_event_ids | uploading_event_ids
-                )
-                logger.debug(f" Total undownloaded events: {len(missing_event_ids)}")
+        missing_event_ids = set(unifi_events.keys()) - (db_event_ids | downloading_event_ids | uploading_event_ids)
 
         def wanted_event_type(event_id):
             event = unifi_events[event_id]
@@ -105,25 +98,49 @@ class MissingEventChecker:
             return True
 
         wanted_event_ids = set(filter(wanted_event_type, missing_event_ids))
-                logger.debug(f" Undownloaded events of wanted types: {len(wanted_event_ids)}")
 
-                if len(wanted_event_ids) > 20:
-                    logger.warning(f" Adding {len(wanted_event_ids)} missing events to backup queue")
+        return [unifi_events[id] for id in wanted_event_ids]
+
+    async def ignore_missing(self):
+        """Ignore missing events by adding them to the event table."""
+        wanted_events = await self._get_missing_events()
+
+        logger.info(f" Ignoring {len(wanted_events)} missing events")
+
+        for event in wanted_events:
+            logger.extra_debug(f"Ignoring event '{event.id}'")
+            await self._db.execute(
+                "INSERT INTO events VALUES "
+                f"('{event.id}', '{event.type}', '{event.camera_id}',"
+                f"'{event.start.timestamp()}', '{event.end.timestamp()}')"
+            )
+        await self._db.commit()
+
+    async def start(self):
+        """Main loop."""
+        logger.info("Starting Missing Event Checker")
+        while True:
+            try:
+                logger.extra_debug("Running check for missing events...")
+
+                wanted_events = await self._get_missing_events()
+
+                logger.debug(f" Undownloaded events of wanted types: {len(wanted_events)}")
+
+                if len(wanted_events) > 20:
+                    logger.warning(f" Adding {len(wanted_events)} missing events to backup queue")
                     missing_logger = logger.extra_debug
                 else:
                     missing_logger = logger.warning
 
-                for event_id in wanted_event_ids:
-                    event = unifi_events[event_id]
+                for event in wanted_events:
                     if event.type != EventType.SMART_DETECT:
-                        missing_logger(
-                            f" Adding missing event to backup queue: {event.id} ({event.type})"
-                            f" ({event.start.strftime('%Y-%m-%dT%H-%M-%S')} -"
-                            f" {event.end.strftime('%Y-%m-%dT%H-%M-%S')})"
-                        )
+                        event_name = f"{event.id} ({event.type})"
                     else:
+                        event_name = f"{event.id} ({', '.join(event.smart_detect_types)})"
+
                     missing_logger(
-                            f" Adding missing event to backup queue: {event.id} ({', '.join(event.smart_detect_types)})"
+                        f" Adding missing event to backup queue: {event_name}"
                         f" ({event.start.strftime('%Y-%m-%dT%H-%M-%S')} -"
                         f" {event.end.strftime('%Y-%m-%dT%H-%M-%S')})"
                     )

unifi_protect_backup/unifi_protect_backup_core.py

@@ -66,6 +66,7 @@ class UnifiProtectBackup:
         download_buffer_size: int,
         purge_interval: str,
         apprise_notifiers: str,
+        skip_missing: bool,
         sqlite_path: str = "events.sqlite",
         color_logging=False,
         port: int = 443,
@@ -93,6 +94,7 @@ class UnifiProtectBackup:
             download_buffer_size (int): How many bytes big the download buffer should be
             purge_interval (str): How often to check for files to delete
             apprise_notifiers (str): Apprise URIs for notifications
+            skip_missing (bool): If initial missing events should be ignored
             sqlite_path (str): Path where to find/create sqlite database
             color_logging (bool): Whether to add color to logging output or not
         """
@@ -123,6 +125,7 @@ class UnifiProtectBackup:
         logger.debug(f"  download_buffer_size={human_readable_size(download_buffer_size)}")
         logger.debug(f"  {purge_interval=}")
         logger.debug(f"  {apprise_notifiers=}")
+        logger.debug(f"  {skip_missing=}")
 
         self.rclone_destination = rclone_destination
         self.retention = parse_rclone_retention(retention)
@@ -152,6 +155,7 @@ class UnifiProtectBackup:
         self._db = None
         self._download_buffer_size = download_buffer_size
         self._purge_interval = parse_rclone_retention(purge_interval)
+        self._skip_missing = skip_missing
 
     async def start(self):
         """Bootstrap the backup process and kick off the main loop.
@@ -245,6 +249,9 @@ class UnifiProtectBackup:
                 self.detection_types,
                 self.ignore_cameras,
             )
+            if self._skip_missing:
+                logger.info("Ignoring missing events")
+                await missing.ignore_missing()
             tasks.append(missing.start())
 
             logger.info("Starting Tasks...")