Bump version: 0.2.1 → 0.3.0

- Remove mention od docs since those were removed
- Clarified how to run the application via poetry

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.2.1
+current_version = 0.3.0
 commit = True
 tag = True
 

.github/workflows/dev.yml

@@ -55,6 +55,7 @@ jobs:
   dev_container:
     name: Create dev container
     runs-on: ubuntu-20.04
+    if: github.event_name != 'pull_request'
     strategy:
       matrix:
         python-versions: [3.9]

CHANGELOG.md

@@ -4,6 +4,15 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.3.0] - 2022-02-22
+### Added
+- New CLI argument for passing CLI arguments directly to `rclone`.
+
+### Fixed
+- A new camera getting added while running no longer crashes the application.
+- A timeout during download now correctly retries the download instead of
+  abandoning the event.
+
 ## [0.2.1] - 2022-02-21
 ### Fixed
 - Retry logging formatting

CONTRIBUTING.md

@@ -59,7 +59,8 @@ Ready to contribute? Here's how to set up `unifi-protect-backup` for local devel
 4. Install dependencies and start your virtualenv:
 
     ```
-    $ poetry install -E test -E doc -E dev
+    $ poetry install -E test -E dev
+    $ poetry shell
     ```
 
 5. Create a branch for local development:
@@ -70,14 +71,21 @@ Ready to contribute? Here's how to set up `unifi-protect-backup` for local devel
 
     Now you can make your changes locally.
 
-6. When you're done making changes, check that your changes pass the
+6. To run `unifi-protect-backup` while developing you will need to either
+   be inside the `poetry shell` virtualenv or run it via poetry:
+
+   ```
+   $ poetry run unifi-protect-backup {args}
+   ```
+
+7. When you're done making changes, check that your changes pass the
    tests, including testing other Python versions, with tox:
 
     ```
     $ poetry run tox
     ```
 
-7. Commit your changes and push your branch to GitHub:
+8. Commit your changes and push your branch to GitHub:
 
     ```
     $ git add .
@@ -85,7 +93,7 @@ Ready to contribute? Here's how to set up `unifi-protect-backup` for local devel
     $ git push origin name-of-your-bugfix-or-feature
     ```
 
-8. Submit a pull request through the GitHub website.
+9. Submit a pull request through the GitHub website.
 
 ## Pull Request Guidelines
 
@@ -93,8 +101,8 @@ Before you submit a pull request, check that it meets these guidelines:
 
 1. The pull request should include tests.
 2. If the pull request adds functionality, the docs should be updated. Put
-   your new functionality into a function with a docstring, and add the
-   feature to the list in README.md.
+   your new functionality into a function with a docstring. If adding a CLI
+   option, you should update the "usage" in README.md.
 3. The pull request should work for Python 3.9. Check
    https://github.com/ep1cman/unifi-protect-backup/actions
    and make sure that the tests pass for all supported Python versions.
@@ -120,4 +128,5 @@ $ git push
 $ git push --tags
 ```
 
-GitHub Actions will then deploy to PyPI if tests pass.
+GitHub Actions will then deploy to PyPI, produce a GitHub release, and a container
+build if tests pass.

Dockerfile

@@ -5,7 +5,7 @@ FROM python:3.9-alpine
 
 WORKDIR /app
 RUN apk add gcc musl-dev zlib-dev jpeg-dev rclone
-COPY dist/unifi-protect-backup-0.2.1.tar.gz sdist.tar.gz
+COPY dist/unifi-protect-backup-0.3.0.tar.gz sdist.tar.gz
 RUN pip install sdist.tar.gz
 
 ENV UFP_USERNAME=unifi_protect_user

README.md

@@ -64,6 +64,15 @@ Options:
                                   of `rclone`
                                   (https://rclone.org/filtering/#max-age-don-
                                   t-transfer-any-file-older-than-this)
+  --rclone-args TEXT              Optional arguments which are directly passed
+                                  to `rclone rcat`. These can by used to set
+                                  parameters such as the bandwidth limit used
+                                  when pushing the files to the rclone
+                                  destination, e.g., '--bwlimit=500k'. Please
+                                  see the `rclone` documentation for the full
+                                  set of arguments it supports
+                                  (https://rclone.org/docs/). Please use
+                                  responsibly.
   --ignore-camera TEXT            IDs of cameras for which events should not
                                   be backed up. Use multiple times to ignore
                                   multiple IDs. If being set as an environment
@@ -106,6 +115,7 @@ always take priority over environment variables):
 - `UFP_SSL_VERIFY`
 - `RCLONE_RETENTION`
 - `RCLONE_DESTINATION`
+- `RCLONE_ARGS`
 - `IGNORE_CAMERAS`
 
 ## Docker Container

pyproject.toml

@@ -1,7 +1,7 @@
 [tool]
 [tool.poetry]
 name = "unifi-protect-backup"
-version = "0.2.1"
+version = "0.3.0"
 homepage = "https://github.com/ep1cman/unifi-protect-backup"
 description = "Python tool to backup unifi event clips in realtime."
 authors = ["sebastian.goscik <sebastian@goscik.com>"]

unifi_protect_backup/__init__.py

@@ -2,6 +2,6 @@
 
 __author__ = """sebastian.goscik"""
 __email__ = 'sebastian@goscik.com'
-__version__ = '0.2.1'
+__version__ = '0.3.0'
 
 from .unifi_protect_backup import UnifiProtectBackup

unifi_protect_backup/cli.py

@@ -32,6 +32,13 @@ from unifi_protect_backup import UnifiProtectBackup
     help="How long should event clips be backed up for. Format as per the `--max-age` argument of "
     "`rclone` (https://rclone.org/filtering/#max-age-don-t-transfer-any-file-older-than-this)",
 )
+@click.option(
+    '--rclone-args',
+    default='',
+    envvar='RCLONE_ARGS',
+    help="Optional extra arguments to pass to `rclone rcat` directly. Common usage for this would "
+    "be to set a bandwidth limit, for example.",
+)
 @click.option(
     '--ignore-camera',
     'ignore_cameras',

unifi_protect_backup/unifi_protect_backup.py

@@ -3,11 +3,12 @@ import asyncio
 import logging
 import pathlib
 import shutil
+from asyncio.exceptions import TimeoutError
 from typing import Callable, List, Optional
 
 import aiocron
-import aiohttp
-from pyunifiprotect import ProtectApiClient
+from aiohttp.client_exceptions import ClientPayloadError
+from pyunifiprotect import NvrError, ProtectApiClient
 from pyunifiprotect.data.nvr import Event
 from pyunifiprotect.data.types import EventType, ModelType
 from pyunifiprotect.data.websocket import WSAction, WSSubscriptionMessage
@@ -168,6 +169,7 @@ class UnifiProtectBackup:
         retention (str): How long should event clips be backed up for. Format as per the
                          `--max-age` argument of `rclone`
                          (https://rclone.org/filtering/#max-age-don-t-transfer-any-file-older-than-this)
+        rclone_args (str): Extra args passed directly to `rclone rcat`.
         ignore_cameras (List[str]): List of camera IDs for which to not backup events
         verbose (int): How verbose to setup logging, see :func:`setup_logging` for details.
         _download_queue (asyncio.Queue): Queue of events that need to be backed up
@@ -182,6 +184,7 @@ class UnifiProtectBackup:
         verify_ssl: bool,
         rclone_destination: str,
         retention: str,
+        rclone_args: str,
         ignore_cameras: List[str],
         verbose: int,
         port: int = 443,
@@ -200,6 +203,8 @@ class UnifiProtectBackup:
             retention (str): How long should event clips be backed up for. Format as per the
                             `--max-age` argument of `rclone`
                             (https://rclone.org/filtering/#max-age-don-t-transfer-any-file-older-than-this)
+            rclone_args (str): A bandwidth limit which is passed to the `--bwlimit` argument of
+                                   `rclone` (https://rclone.org/docs/#bwlimit-bandwidth-spec)
             ignore_cameras (List[str]): List of camera IDs for which to not backup events
             verbose (int): How verbose to setup logging, see :func:`setup_logging` for details.
         """
@@ -217,11 +222,13 @@ class UnifiProtectBackup:
         logger.debug(f"  {verify_ssl=}")
         logger.debug(f"  {rclone_destination=}")
         logger.debug(f"  {retention=}")
+        logger.debug(f"  {rclone_args=}")
         logger.debug(f"  {ignore_cameras=}")
         logger.debug(f"  {verbose=}")
 
         self.rclone_destination = rclone_destination
         self.retention = retention
+        self.rclone_args = rclone_args
 
         self._protect = ProtectApiClient(
             address,
@@ -250,6 +257,8 @@ class UnifiProtectBackup:
         # Start the pyunifiprotect connection by calling `update`
         logger.info("Connecting to Unifi Protect...")
         await self._protect.update()
+
+        # Get a mapping of camera ids -> names
         logger.info("Found cameras:")
         for camera in self._protect.bootstrap.cameras.values():
             logger.info(f" - {camera.id}: {camera.name}")
@@ -354,18 +363,17 @@ class UnifiProtectBackup:
 
         """
         while True:
-            event = await self._download_queue.get()
-            destination = self.generate_file_path(event)
-
-            logger.info(f"Backing up event: {event.id}")
-            logger.debug(f"Remaining Queue: {self._download_queue.qsize()}")
-            logger.debug(f"  Camera: {self._protect.bootstrap.cameras[event.camera_id].name}")
-            logger.debug(f"  Type: {event.type}")
-            logger.debug(f"  Start: {event.start.strftime('%Y-%m-%dT%H-%M-%S')}")
-            logger.debug(f"  End: {event.end.strftime('%Y-%m-%dT%H-%M-%S')}")
-            logger.debug(f"  Duration: {event.end-event.start}")
-
             try:
+                event = await self._download_queue.get()
+
+                logger.info(f"Backing up event: {event.id}")
+                logger.debug(f"Remaining Queue: {self._download_queue.qsize()}")
+                logger.debug(f"  Camera: {await self._get_camera_name(event.camera_id)}")
+                logger.debug(f"  Type: {event.type}")
+                logger.debug(f"  Start: {event.start.strftime('%Y-%m-%dT%H-%M-%S')}")
+                logger.debug(f"  End: {event.end.strftime('%Y-%m-%dT%H-%M-%S')}")
+                logger.debug(f"  Duration: {event.end-event.start}")
+
                 # Download video
                 logger.debug("  Downloading video...")
                 for x in range(5):
@@ -373,7 +381,7 @@ class UnifiProtectBackup:
                         video = await self._protect.get_camera_video(event.camera_id, event.start, event.end)
                         assert isinstance(video, bytes)
                         break
-                    except (AssertionError, aiohttp.client_exceptions.ClientPayloadError) as e:
+                    except (AssertionError, ClientPayloadError, TimeoutError) as e:
                         logger.warn(f"    Failed download attempt {x+1}, retying in 1s")
                         logger.exception(e)
                         await asyncio.sleep(1)
@@ -381,12 +389,14 @@ class UnifiProtectBackup:
                     logger.warn(f"Download failed after 5 attempts, abandoning event {event.id}:")
                     continue
 
+                destination = await self.generate_file_path(event)
+
                 logger.debug("  Uploading video via rclone...")
                 logger.debug(f"    To: {destination}")
                 logger.debug(f"    Size: {human_readable_size(len(video))}")
                 for x in range(5):
                     try:
-                        await self._upload_video(video, destination)
+                        await self._upload_video(video, destination, self.rclone_args)
                         break
                     except RcloneException as e:
                         logger.warn(f"    Failed upload attempt {x+1}, retying in 1s")
@@ -402,7 +412,7 @@ class UnifiProtectBackup:
                 logger.warn(f"Unexpected exception occurred, abandoning event {event.id}:")
                 logger.exception(e)
 
-    async def _upload_video(self, video: bytes, destination: pathlib.Path):
+    async def _upload_video(self, video: bytes, destination: pathlib.Path, rclone_args: str):
         """Upload video using rclone.
 
         In order to avoid writing to disk, the video file data is piped directly
@@ -411,11 +421,12 @@ class UnifiProtectBackup:
         Args:
             video (bytes): The data to be written to the file
             destination (pathlib.Path): Where rclone should write the file
+            rclone_args (str): Optional extra arguments to pass to `rclone`
 
         Raises:
             RuntimeError: If rclone returns a non-zero exit code
         """
-        cmd = f"rclone rcat -vv '{destination}'"
+        cmd = f"rclone rcat -vv {rclone_args} '{destination}'"
         proc = await asyncio.create_subprocess_shell(
             cmd,
             stdin=asyncio.subprocess.PIPE,
@@ -429,7 +440,7 @@ class UnifiProtectBackup:
         else:
             raise RcloneException(stdout.decode(), stderr.decode(), proc.returncode)
 
-    def generate_file_path(self, event: Event) -> pathlib.Path:
+    async def generate_file_path(self, event: Event) -> pathlib.Path:
         """Generates the rclone destination path for the provided event.
 
         Generates paths in the following structure:
@@ -448,7 +459,7 @@ class UnifiProtectBackup:
         """
         path = pathlib.Path(self.rclone_destination)
         assert isinstance(event.camera_id, str)
-        path /= self._protect.bootstrap.cameras[event.camera_id].name  # directory per camera
+        path /= await self._get_camera_name(event.camera_id)  # directory per camera
         path /= event.start.strftime("%Y-%m-%d")  # Directory per day
 
         file_name = f"{event.start.strftime('%Y-%m-%dT%H-%M-%S')} {event.type}"
@@ -461,3 +472,20 @@ class UnifiProtectBackup:
         path /= file_name
 
         return path
+
+    async def _get_camera_name(self, id: str):
+        try:
+            return self._protect.bootstrap.cameras[id].name
+        except KeyError:
+            # Refresh cameras
+            logger.debug(f"Unknown camera id: '{id}', checking API")
+
+            try:
+                await self._protect.update(force=True)
+            except NvrError:
+                logger.debug(f"Unknown camera id: '{id}'")
+                raise
+
+            name = self._protect.bootstrap.cameras[id].name
+            logger.debug(f"Found camera - {id}: {name}")
+            return name