Bump version: 0.6.0 → 0.7.0

Add ability to change clip file structure via template

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.4.0
+current_version = 0.7.0
 commit = True
 tag = True
 

.gitignore

@@ -113,5 +113,8 @@ ENV/
 # mkdocs build dir
 site/
 
+# Docker mounted volumes
+config/
+data/
 
 .envrc

CHANGELOG.md

@@ -4,7 +4,43 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
-## [0.4.0] - 2022-02-24
+## [0.7.0] - 2022-03-26
+### Added
+- Added a the ability to change the way the clip files are structured via a template string.
+### Fixed
+- Fixed issue where event types without clips would attempt (and fail1) to download clips
+- Drastically reduced the size of the docker container
+- Fixed typos in the documentation
+- Some dev dependencies are now not installed as default
+
+## [0.6.0] - 2022-03-18
+### Added
+- Support for doorbell ring events
+- `detection_types` parameter to limit which kinds of events are backed up
+### Fixed
+- Actually fixed timestamps this time.
+
+## [0.5.3] - 2022-03-11
+### Fixed
+- Timestamps in filenames and logging now show time in the timezone of the NVR not UTC
+
+## [0.5.2] - 2022-03-10
+### Fixed
+- rclone delete command now works as expected on windows when spaces are in the file path
+- Dockerfile now allows setting of user and group to run as, as well as a default config
+
+## [0.5.1] - 2022-03-07
+### Fixed
+- rclone command now works as expected on windows when spaces are in the file path
+
+## [0.5.0] - 2022-03-06
+### Added
+- If `ffprobe` is available, the downloaded clips length is checked and logged
+### Fixed
+- A time delay has been added before downloading clips to try to resolve an issue where
+  downloaded clips were too short
+
+## [0.4.0] - 2022-03-05
 ### Added
 - A `--version` command line option to show the tools version
 ### Fixed

Dockerfile

@@ -1,24 +1,51 @@
 # To build run:
 # $ poetry build
 # $ docker build -t ghcr.io/ep1cman/unifi-protect-backup .
-FROM python:3.9-alpine
+
+FROM ghcr.io/linuxserver/baseimage-alpine:3.15
+
+LABEL maintainer="ep1cman"
 
 WORKDIR /app
-RUN apk add gcc musl-dev zlib-dev jpeg-dev rclone
-COPY dist/unifi-protect-backup-0.4.0.tar.gz sdist.tar.gz
-RUN pip install sdist.tar.gz
 
+COPY dist/unifi-protect-backup-0.7.0.tar.gz sdist.tar.gz
+
+RUN \
+    echo "**** install build packages ****" && \
+    apk add --no-cache --virtual=build-dependencies \
+    gcc \
+    musl-dev \
+    jpeg-dev \
+    zlib-dev \
+    python3-dev && \
+    echo "**** install packages ****" && \
+    apk add --no-cache \
+    rclone \
+    ffmpeg \
+    py3-pip \
+    python3 && \
+    echo "**** install unifi-protect-backup ****" && \
+    pip install --no-cache-dir sdist.tar.gz && \
+    echo "**** cleanup ****" && \
+    apk del --purge \
+    build-dependencies && \
+    rm -rf \
+    /tmp/* \
+    /app/sdist.tar.gz
+
+# Settings
 ENV UFP_USERNAME=unifi_protect_user
 ENV UFP_PASSWORD=unifi_protect_password
 ENV UFP_ADDRESS=127.0.0.1
 ENV UFP_PORT=443
 ENV UFP_SSL_VERIFY=true
 ENV RCLONE_RETENTION=7d
-ENV RCLONE_DESTINATION=my_remote:/unifi_protect_backup
+ENV RCLONE_DESTINATION=local:/data
 ENV VERBOSITY="v"
 ENV TZ=UTC
 ENV IGNORE_CAMERAS=""
 
-VOLUME [ "/root/.config/rclone/" ]
+COPY docker_root/ /
 
-CMD ["sh", "-c", "unifi-protect-backup -${VERBOSITY}"]
+VOLUME [ "/config" ]
+VOLUME [ "/data" ]

README.md

@@ -28,7 +28,7 @@ retention period.
 
 ## Requirements
 - Python 3.9+
-- Unifi Protect version 1.20 or higher (as per [`pyunifiproect`](https://github.com/briis/pyunifiprotect))
+- Unifi Protect version 1.20 or higher (as per [`pyunifiprotect`](https://github.com/briis/pyunifiprotect))
 - `rclone` installed with at least one remote configured.
 
 ## Installation
@@ -36,6 +36,7 @@ retention period.
 1. Install `rclone`. Instructions for your platform can be found here: https://rclone.org/install/#quickstart
 2. Configure the `rclone` remote you want to backup to. Instructions can be found here: https://rclone.org/docs/#configure
 3. `pip install unifi-protect-backup`
+4. Optional: Install `ffprobe` so that `unifi-protect-backup` can check the length of the clips it downloads
 
 ## Usage
 
@@ -48,14 +49,17 @@ Usage: unifi-protect-backup [OPTIONS]
   A Python based tool for backing up Unifi Protect event clips as they occur.
 
 Options:
+  --version                       Show the version and exit.
   --address TEXT                  Address of Unifi Protect instance
                                   [required]
-  --port INTEGER                  Port of Unifi Protect instance
+  --port INTEGER                  Port of Unifi Protect instance  [default:
+                                  443]
   --username TEXT                 Username to login to Unifi Protect instance
                                   [required]
   --password TEXT                 Password for Unifi Protect user  [required]
   --verify-ssl / --no-verify-ssl  Set if you do not have a valid HTTPS
-                                  Certificate for your instance
+                                  Certificate for your instance  [default:
+                                  verify-ssl]
   --rclone-destination TEXT       `rclone` destination path in the format
                                   {rclone remote}:{path on remote}. E.g.
                                   `gdrive:/backups/unifi_protect`  [required]
@@ -64,20 +68,25 @@ Options:
                                   of `rclone`
                                   (https://rclone.org/filtering/#max-age-don-
                                   t-transfer-any-file-older-than-this)
-  --rclone-args TEXT              Optional arguments which are directly passed
-                                  to `rclone rcat`. These can by used to set
-                                  parameters such as the bandwidth limit used
-                                  when pushing the files to the rclone
-                                  destination, e.g., '--bwlimit=500k'. Please
-                                  see the `rclone` documentation for the full
-                                  set of arguments it supports
-                                  (https://rclone.org/docs/). Please use
-                                  responsibly.
+                                  [default: 7d]
+  --rclone-args TEXT              Optional extra arguments to pass to `rclone
+                                  rcat` directly. Common usage for this would
+                                  be to set a bandwidth limit, for example.
+  --detection-types TEXT          A comma separated list of which types of
+                                  detections to backup. Valid options are:
+                                  `motion`, `person`, `vehicle`, `ring`
+                                  [default: motion,person,vehicle,ring]
   --ignore-camera TEXT            IDs of cameras for which events should not
                                   be backed up. Use multiple times to ignore
                                   multiple IDs. If being set as an environment
                                   variable the IDs should be separated by
                                   whitespace.
+  --file-structure-format TEXT    A Python format string used to generate the
+                                  file structure/name on the rclone remote.For
+                                  details of the fields available, see the
+                                  projects `README.md` file.  [default: {camer
+                                  a_name}/{event.start:%Y-%m-%d}/{event.end:%Y
+                                  -%m-%dT%H-%M-%S} {detection_type}.mp4]
   -v, --verbose                   How verbose the logging output should be.
 
                                       None: Only log info messages created by
@@ -117,29 +126,70 @@ always take priority over environment variables):
 - `RCLONE_DESTINATION`
 - `RCLONE_ARGS`
 - `IGNORE_CAMERAS`
+- `DETECTION_TYPES`
+- `FILE_STRUCTURE_FORMAT`
+
+## File path formatting
+
+By default, the application will save clips in the following structure on the provided rclone remote:
+```
+{camera_name}/{event.start:%Y-%m-%d}/{event.end:%Y-%m-%dT%H-%M-%S} {detection_type}.mp4
+```
+If you wish for the clips to be structured differently you can do this using the `--file-structure-format`
+option. It uses standard [python format string syntax](https://docs.python.org/3/library/string.html#formatstrings).
+
+The following fields are provided to the format string:
+  - *event:* The `Event` object as per https://github.com/briis/pyunifiprotect/blob/master/pyunifiprotect/data/nvr.py
+  - *duration_seconds:* The duration of the event in seconds
+  - *detection_type:* A nicely formatted list of the event detection type and the smart detection types (if any)
+  - *camera_name:* The name of the camera that generated this event
+
+You can optionally format the `event.start`/`event.end` timestamps as per the [`strftime` format](https://docs.python.org/3/library/datetime.html#strftime-strptime-behavior) by appending it after a `:` e.g to get just the date without the time: `{event.start:%Y-%m-%d}`
+
 
 ## Docker Container
 You can run this tool as a container if you prefer with the following command.
 Remember to change the variable to make your setup.
 
+
+### Backing up locally
+By default, if no rclone config is provided clips will be backed up to `/data`.
+
 ```
 docker run \
   -e UFP_USERNAME='USERNAME' \
   -e UFP_PASSWORD='PASSWORD' \
   -e UFP_ADDRESS='UNIFI_PROTECT_IP' \
   -e UFP_SSL_VERIFY='false' \
-  -e RCLONE_DESTINATION='my_remote:/unifi_protect_backup' \
-  -v '/path/to/rclone.conf':'/root/.config/rclone/rclone.conf' \
+  -v '/path/to/save/clips':'/data' \
   ghcr.io/ep1cman/unifi-protect-backup
 ```
+
+### Backing up to cloud storage
+In order to backup to cloud storage you need to provide a `rclone.conf` file.
+
 If you do not already have a `rclone.conf` file you can create one as follows:
 ```
-$ docker run -it --rm -v $PWD:/root/.config/rclone/ ghcr.io/ep1cman/unifi-protect-backup rclone config
+$ docker run -it --rm -v $PWD:/root/.config/rclone rclone/rclone config
+```
+Follow the interactive configuration proceed, this will create a `rclone.conf`
+file in your current directory.
+
+Finally start the container:
+```
+docker run \
+  -e UFP_USERNAME='USERNAME' \
+  -e UFP_PASSWORD='PASSWORD' \
+  -e UFP_ADDRESS='UNIFI_PROTECT_IP' \
+  -e UFP_SSL_VERIFY='false' \
+  -e RCLONE_DESTINATION='my_remote:/unifi_protect_backup' \
+  -v '/path/to/save/clips':'/data' \
+  -v `/path/to/rclone.conf':'/config/rclone.conf'
+  ghcr.io/ep1cman/unifi-protect-backup
 ```
-This will create a `rclone.conf` file in your current directory
 
 ## Credits
 
-- Heavily utilises [`pyunifiproect`](https://github.com/briis/pyunifiprotect) by [@briis](https://github.com/briis/)
+- Heavily utilises [`pyunifiprotect`](https://github.com/briis/pyunifiprotect) by [@briis](https://github.com/briis/)
 - All the cloud functionality is provided by [`rclone`](https://rclone.org/)
 - This package was created with [Cookiecutter](https://github.com/audreyr/cookiecutter) and the [waynerv/cookiecutter-pypackage](https://github.com/waynerv/cookiecutter-pypackage) project template.

docker_root/defaults/rclone.conf

@@ -0,0 +1,2 @@
+[local]
+type = local

docker_root/etc/cont-init.d/30-config

@@ -0,0 +1,23 @@
+#!/usr/bin/with-contenv bash
+
+mkdir -p /config/rclone
+
+# For backwards compatibility
+[[ -f "/root/.config/rclone/rclone.conf" ]] && \
+    echo "DEPRECATED: Copying rclone conf from /root/.config/rclone/rclone.conf, please change your mount to /config/rclone.conf"
+    cp \
+        /root/.config/rclone/rclone.conf \
+        /config/rclone/rclone.conf
+
+# default config file
+[[ ! -f "/config/rclone/rclone.conf" ]] && \
+    mkdir -p /config/rclone && \
+    cp \
+        /defaults/rclone.conf \
+        /config/rclone/rclone.conf
+
+chown -R abc:abc \
+  /config
+
+chown -R abc:abc \
+  /data

docker_root/etc/services.d/unifi-protect-backup/run

@@ -0,0 +1,6 @@
+#!/usr/bin/with-contenv bash
+
+
+export RCLONE_CONFIG=/config/rclone/rclone.conf
+exec \
+    s6-setuidgid abc unifi-protect-backup -${VERBOSITY}

poetry.lock

@@ -52,6 +52,28 @@ python-versions = ">=3.6"
 [package.dependencies]
 frozenlist = ">=1.1.0"
 
+[[package]]
+name = "appnope"
+version = "0.1.2"
+description = "Disable App Nap on macOS >= 10.9"
+category = "main"
+optional = true
+python-versions = "*"
+
+[[package]]
+name = "asttokens"
+version = "2.0.5"
+description = "Annotate AST trees with source code positions"
+category = "main"
+optional = true
+python-versions = "*"
+
+[package.dependencies]
+six = "*"
+
+[package.extras]
+test = ["astroid", "pytest"]
+
 [[package]]
 name = "async-timeout"
 version = "4.0.2"
@@ -90,6 +112,14 @@ docs = ["furo", "sphinx", "zope.interface", "sphinx-notfound-page"]
 tests = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "mypy", "pytest-mypy-plugins", "zope.interface", "cloudpickle"]
 tests_no_zope = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "mypy", "pytest-mypy-plugins", "cloudpickle"]
 
+[[package]]
+name = "backcall"
+version = "0.2.0"
+description = "Specifications for callback functions passed in to an API"
+category = "main"
+optional = true
+python-versions = "*"
+
 [[package]]
 name = "black"
 version = "21.12b0"
@@ -235,6 +265,14 @@ sdist = ["setuptools_rust (>=0.11.4)"]
 ssh = ["bcrypt (>=3.1.5)"]
 test = ["pytest (>=6.2.0)", "pytest-cov", "pytest-subtests", "pytest-xdist", "pretend", "iso8601", "pytz", "hypothesis (>=1.11.4,!=3.79.2)"]
 
+[[package]]
+name = "decorator"
+version = "5.1.1"
+description = "Decorators for Humans"
+category = "main"
+optional = true
+python-versions = ">=3.5"
+
 [[package]]
 name = "distlib"
 version = "0.3.4"
@@ -251,6 +289,14 @@ category = "main"
 optional = true
 python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*"
 
+[[package]]
+name = "executing"
+version = "0.8.3"
+description = "Get the currently executing AST node of a frame, and other information"
+category = "main"
+optional = true
+python-versions = "*"
+
 [[package]]
 name = "filelock"
 version = "3.6.0"
@@ -339,6 +385,54 @@ category = "main"
 optional = true
 python-versions = "*"
 
+[[package]]
+name = "ipdb"
+version = "0.13.9"
+description = "IPython-enabled pdb"
+category = "main"
+optional = true
+python-versions = ">=2.7"
+
+[package.dependencies]
+decorator = {version = "*", markers = "python_version > \"3.6\""}
+ipython = {version = ">=7.17.0", markers = "python_version > \"3.6\""}
+toml = {version = ">=0.10.2", markers = "python_version > \"3.6\""}
+
+[[package]]
+name = "ipython"
+version = "8.1.1"
+description = "IPython: Productive Interactive Computing"
+category = "main"
+optional = true
+python-versions = ">=3.8"
+
+[package.dependencies]
+appnope = {version = "*", markers = "sys_platform == \"darwin\""}
+backcall = "*"
+colorama = {version = "*", markers = "sys_platform == \"win32\""}
+decorator = "*"
+jedi = ">=0.16"
+matplotlib-inline = "*"
+pexpect = {version = ">4.3", markers = "sys_platform != \"win32\""}
+pickleshare = "*"
+prompt-toolkit = ">=2.0.0,<3.0.0 || >3.0.0,<3.0.1 || >3.0.1,<3.1.0"
+pygments = ">=2.4.0"
+stack-data = "*"
+traitlets = ">=5"
+
+[package.extras]
+all = ["black", "Sphinx (>=1.3)", "ipykernel", "nbconvert", "nbformat", "ipywidgets", "notebook", "ipyparallel", "qtconsole", "curio", "matplotlib (!=3.2.0)", "numpy (>=1.19)", "pandas", "pytest", "testpath", "trio", "pytest-asyncio"]
+black = ["black"]
+doc = ["Sphinx (>=1.3)"]
+kernel = ["ipykernel"]
+nbconvert = ["nbconvert"]
+nbformat = ["nbformat"]
+notebook = ["ipywidgets", "notebook"]
+parallel = ["ipyparallel"]
+qtconsole = ["qtconsole"]
+test = ["pytest", "pytest-asyncio", "testpath"]
+test_extra = ["curio", "matplotlib (!=3.2.0)", "nbformat", "numpy (>=1.19)", "pandas", "pytest", "testpath", "trio"]
+
 [[package]]
 name = "isort"
 version = "5.10.1"
@@ -353,6 +447,21 @@ requirements_deprecated_finder = ["pipreqs", "pip-api"]
 colors = ["colorama (>=0.4.3,<0.5.0)"]
 plugins = ["setuptools"]
 
+[[package]]
+name = "jedi"
+version = "0.18.1"
+description = "An autocompletion tool for Python that can be used for text editors."
+category = "main"
+optional = true
+python-versions = ">=3.6"
+
+[package.dependencies]
+parso = ">=0.8.0,<0.9.0"
+
+[package.extras]
+qa = ["flake8 (==3.8.3)", "mypy (==0.782)"]
+testing = ["Django (<3.1)", "colorama", "docopt", "pytest (<7.0.0)"]
+
 [[package]]
 name = "jeepney"
 version = "0.7.1"
@@ -383,6 +492,17 @@ SecretStorage = {version = ">=3.2", markers = "sys_platform == \"linux\""}
 docs = ["sphinx", "jaraco.packaging (>=8.2)", "rst.linker (>=1.9)", "jaraco.tidelift (>=1.4)"]
 testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest-cov", "pytest-enabler (>=1.0.1)", "pytest-black (>=0.3.7)", "pytest-mypy"]
 
+[[package]]
+name = "matplotlib-inline"
+version = "0.1.3"
+description = "Inline Matplotlib backend for Jupyter"
+category = "main"
+optional = true
+python-versions = ">=3.5"
+
+[package.dependencies]
+traitlets = "*"
+
 [[package]]
 name = "mccabe"
 version = "0.6.1"
@@ -443,6 +563,18 @@ python-versions = ">=3.6"
 [package.dependencies]
 pyparsing = ">=2.0.2,<3.0.5 || >3.0.5"
 
+[[package]]
+name = "parso"
+version = "0.8.3"
+description = "A Python Parser"
+category = "main"
+optional = true
+python-versions = ">=3.6"
+
+[package.extras]
+qa = ["flake8 (==3.8.3)", "mypy (==0.782)"]
+testing = ["docopt", "pytest (<6.0.0)"]
+
 [[package]]
 name = "pathspec"
 version = "0.9.0"
@@ -451,6 +583,25 @@ category = "main"
 optional = true
 python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7"
 
+[[package]]
+name = "pexpect"
+version = "4.8.0"
+description = "Pexpect allows easy control of interactive console applications."
+category = "main"
+optional = true
+python-versions = "*"
+
+[package.dependencies]
+ptyprocess = ">=0.5"
+
+[[package]]
+name = "pickleshare"
+version = "0.7.5"
+description = "Tiny 'shelve'-like database with concurrency support"
+category = "main"
+optional = true
+python-versions = "*"
+
 [[package]]
 name = "pillow"
 version = "9.0.1"
@@ -510,6 +661,36 @@ pyyaml = ">=5.1"
 toml = "*"
 virtualenv = ">=20.0.8"
 
+[[package]]
+name = "prompt-toolkit"
+version = "3.0.28"
+description = "Library for building powerful interactive command lines in Python"
+category = "main"
+optional = true
+python-versions = ">=3.6.2"
+
+[package.dependencies]
+wcwidth = "*"
+
+[[package]]
+name = "ptyprocess"
+version = "0.7.0"
+description = "Run a subprocess in a pseudo terminal"
+category = "main"
+optional = true
+python-versions = "*"
+
+[[package]]
+name = "pure-eval"
+version = "0.2.2"
+description = "Safely evaluate AST nodes without side effects"
+category = "main"
+optional = true
+python-versions = "*"
+
+[package.extras]
+tests = ["pytest"]
+
 [[package]]
 name = "py"
 version = "1.11.0"
@@ -806,6 +987,22 @@ category = "main"
 optional = true
 python-versions = "*"
 
+[[package]]
+name = "stack-data"
+version = "0.2.0"
+description = "Extract data from python stack frames and tracebacks for informative displays"
+category = "main"
+optional = true
+python-versions = "*"
+
+[package.dependencies]
+asttokens = "*"
+executing = "*"
+pure-eval = "*"
+
+[package.extras]
+tests = ["pytest", "typeguard", "pygments", "littleutils", "cython"]
+
 [[package]]
 name = "toml"
 version = "0.10.2"
@@ -874,6 +1071,17 @@ dev = ["py-make (>=0.1.0)", "twine", "wheel"]
 notebook = ["ipywidgets (>=6)"]
 telegram = ["requests"]
 
+[[package]]
+name = "traitlets"
+version = "5.1.1"
+description = "Traitlets Python configuration system"
+category = "main"
+optional = true
+python-versions = ">=3.7"
+
+[package.extras]
+test = ["pytest"]
+
 [[package]]
 name = "twine"
 version = "3.8.0"
@@ -911,6 +1119,22 @@ dev = ["autoflake (>=1.3.1,<2.0.0)", "flake8 (>=3.8.3,<4.0.0)"]
 doc = ["mkdocs (>=1.1.2,<2.0.0)", "mkdocs-material (>=5.4.0,<6.0.0)", "markdown-include (>=0.5.1,<0.6.0)"]
 test = ["shellingham (>=1.3.0,<2.0.0)", "pytest (>=4.4.0,<5.4.0)", "pytest-cov (>=2.10.0,<3.0.0)", "coverage (>=5.2,<6.0)", "pytest-xdist (>=1.32.0,<2.0.0)", "pytest-sugar (>=0.9.4,<0.10.0)", "mypy (==0.910)", "black (>=19.10b0,<20.0b0)", "isort (>=5.0.6,<6.0.0)"]
 
+[[package]]
+name = "types-cryptography"
+version = "3.3.18"
+description = "Typing stubs for cryptography"
+category = "main"
+optional = false
+python-versions = "*"
+
+[[package]]
+name = "types-pytz"
+version = "2021.3.5"
+description = "Typing stubs for pytz"
+category = "main"
+optional = true
+python-versions = "*"
+
 [[package]]
 name = "typing-extensions"
 version = "4.1.1"
@@ -974,6 +1198,14 @@ six = ">=1.9.0,<2"
 docs = ["proselint (>=0.10.2)", "sphinx (>=3)", "sphinx-argparse (>=0.2.5)", "sphinx-rtd-theme (>=0.4.3)", "towncrier (>=21.3)"]
 testing = ["coverage (>=4)", "coverage-enable-subprocess (>=1)", "flaky (>=3)", "pytest (>=4)", "pytest-env (>=0.6.2)", "pytest-freezegun (>=0.4.1)", "pytest-mock (>=2)", "pytest-randomly (>=1)", "pytest-timeout (>=1)", "packaging (>=20.0)"]
 
+[[package]]
+name = "wcwidth"
+version = "0.2.5"
+description = "Measures the displayed width of unicode strings in a terminal"
+category = "main"
+optional = true
+python-versions = "*"
+
 [[package]]
 name = "webencodings"
 version = "0.5.1"
@@ -1007,13 +1239,13 @@ docs = ["sphinx", "jaraco.packaging (>=8.2)", "rst.linker (>=1.9)"]
 testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest-cov", "pytest-enabler (>=1.0.1)", "jaraco.itertools", "func-timeout", "pytest-black (>=0.3.7)", "pytest-mypy"]
 
 [extras]
-dev = ["tox", "pre-commit", "virtualenv", "pip", "twine", "toml", "bump2version", "tox-asdf"]
-test = ["pytest", "black", "isort", "mypy", "flake8", "flake8-docstrings", "pytest-cov"]
+dev = ["tox", "pre-commit", "virtualenv", "pip", "twine", "toml", "bump2version", "tox-asdf", "ipdb"]
+test = ["pytest", "black", "isort", "mypy", "flake8", "flake8-docstrings", "pytest-cov", "types-pytz"]
 
 [metadata]
 lock-version = "1.1"
 python-versions = ">=3.9.0,<4.0"
-content-hash = "486d2f4f9e4d4ac8fd9effb434c192585266838cd27f7d0d2688cd62282ab2d3"
+content-hash = "46d8a3be305d84d94aaba9502d47be0db432269de41f0794abd2f4cb28ac7338"
 
 [metadata.files]
 aiocron = [
@@ -1102,6 +1334,14 @@ aiosignal = [
     {file = "aiosignal-1.2.0-py3-none-any.whl", hash = "sha256:26e62109036cd181df6e6ad646f91f0dcfd05fe16d0cb924138ff2ab75d64e3a"},
     {file = "aiosignal-1.2.0.tar.gz", hash = "sha256:78ed67db6c7b7ced4f98e495e572106d5c432a93e1ddd1bf475e1dc05f5b7df2"},
 ]
+appnope = [
+    {file = "appnope-0.1.2-py2.py3-none-any.whl", hash = "sha256:93aa393e9d6c54c5cd570ccadd8edad61ea0c4b9ea7a01409020c9aa019eb442"},
+    {file = "appnope-0.1.2.tar.gz", hash = "sha256:dd83cd4b5b460958838f6eb3000c660b1f9caf2a5b1de4264e941512f603258a"},
+]
+asttokens = [
+    {file = "asttokens-2.0.5-py2.py3-none-any.whl", hash = "sha256:0844691e88552595a6f4a4281a9f7f79b8dd45ca4ccea82e5e05b4bbdb76705c"},
+    {file = "asttokens-2.0.5.tar.gz", hash = "sha256:9a54c114f02c7a9480d56550932546a3f1fe71d8a02f1bc7ccd0ee3ee35cf4d5"},
+]
 async-timeout = [
     {file = "async-timeout-4.0.2.tar.gz", hash = "sha256:2163e1640ddb52b7a8c80d0a67a08587e5d245cc9c553a74a847056bc2976b15"},
     {file = "async_timeout-4.0.2-py3-none-any.whl", hash = "sha256:8ca1e4fcf50d07413d66d1a5e416e42cfdf5851c981d679a09851a6853383b3c"},
@@ -1120,6 +1360,10 @@ attrs = [
     {file = "attrs-21.4.0-py2.py3-none-any.whl", hash = "sha256:2d27e3784d7a565d36ab851fe94887c5eccd6a463168875832a1be79c82828b4"},
     {file = "attrs-21.4.0.tar.gz", hash = "sha256:626ba8234211db98e869df76230a137c4c40a12d72445c45d5f5b716f076e2fd"},
 ]
+backcall = [
+    {file = "backcall-0.2.0-py2.py3-none-any.whl", hash = "sha256:fbbce6a29f263178a1f7915c1940bde0ec2b2a967566fe1c65c1dfb7422bd255"},
+    {file = "backcall-0.2.0.tar.gz", hash = "sha256:5cbdbf27be5e7cfadb448baf0aa95508f91f2bbc6c6437cd9cd06e2a4c215e1e"},
+]
 black = [
     {file = "black-21.12b0-py3-none-any.whl", hash = "sha256:a615e69ae185e08fdd73e4715e260e2479c861b5740057fde6e8b4e3b7dd589f"},
     {file = "black-21.12b0.tar.gz", hash = "sha256:77b80f693a569e2e527958459634f18df9b0ba2625ba4e0c2d5da5be42e6f2b3"},
@@ -1273,6 +1517,10 @@ cryptography = [
     {file = "cryptography-36.0.1-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:39bdf8e70eee6b1c7b289ec6e5d84d49a6bfa11f8b8646b5b3dfe41219153316"},
     {file = "cryptography-36.0.1.tar.gz", hash = "sha256:53e5c1dc3d7a953de055d77bef2ff607ceef7a2aac0353b5d630ab67f7423638"},
 ]
+decorator = [
+    {file = "decorator-5.1.1-py3-none-any.whl", hash = "sha256:b8c3f85900b9dc423225913c5aace94729fe1fa9763b38939a95226f02d37186"},
+    {file = "decorator-5.1.1.tar.gz", hash = "sha256:637996211036b6385ef91435e4fae22989472f9d571faba8927ba8253acbc330"},
+]
 distlib = [
     {file = "distlib-0.3.4-py2.py3-none-any.whl", hash = "sha256:6564fe0a8f51e734df6333d08b8b94d4ea8ee6b99b5ed50613f731fd4089f34b"},
     {file = "distlib-0.3.4.zip", hash = "sha256:e4b58818180336dc9c529bfb9a0b58728ffc09ad92027a3f30b7cd91e3458579"},
@@ -1281,6 +1529,10 @@ docutils = [
     {file = "docutils-0.18.1-py2.py3-none-any.whl", hash = "sha256:23010f129180089fbcd3bc08cfefccb3b890b0050e1ca00c867036e9d161b98c"},
     {file = "docutils-0.18.1.tar.gz", hash = "sha256:679987caf361a7539d76e584cbeddc311e3aee937877c87346f31debc63e9d06"},
 ]
+executing = [
+    {file = "executing-0.8.3-py2.py3-none-any.whl", hash = "sha256:d1eef132db1b83649a3905ca6dd8897f71ac6f8cac79a7e58a1a09cf137546c9"},
+    {file = "executing-0.8.3.tar.gz", hash = "sha256:c6554e21c6b060590a6d3be4b82fb78f8f0194d809de5ea7df1c093763311501"},
+]
 filelock = [
     {file = "filelock-3.6.0-py3-none-any.whl", hash = "sha256:f8314284bfffbdcfa0ff3d7992b023d4c628ced6feb957351d4c48d059f56bc0"},
     {file = "filelock-3.6.0.tar.gz", hash = "sha256:9cd540a9352e432c7246a48fe4e8712b10acb1df2ad1f30e8c070b82ae1fed85"},
@@ -1370,10 +1622,21 @@ iniconfig = [
     {file = "iniconfig-1.1.1-py2.py3-none-any.whl", hash = "sha256:011e24c64b7f47f6ebd835bb12a743f2fbe9a26d4cecaa7f53bc4f35ee9da8b3"},
     {file = "iniconfig-1.1.1.tar.gz", hash = "sha256:bc3af051d7d14b2ee5ef9969666def0cd1a000e121eaea580d4a313df4b37f32"},
 ]
+ipdb = [
+    {file = "ipdb-0.13.9.tar.gz", hash = "sha256:951bd9a64731c444fd907a5ce268543020086a697f6be08f7cc2c9a752a278c5"},
+]
+ipython = [
+    {file = "ipython-8.1.1-py3-none-any.whl", hash = "sha256:6f56bfaeaa3247aa3b9cd3b8cbab3a9c0abf7428392f97b21902d12b2f42a381"},
+    {file = "ipython-8.1.1.tar.gz", hash = "sha256:8138762243c9b3a3ffcf70b37151a2a35c23d3a29f9743878c33624f4207be3d"},
+]
 isort = [
     {file = "isort-5.10.1-py3-none-any.whl", hash = "sha256:6f62d78e2f89b4500b080fe3a81690850cd254227f27f75c3a0c491a1f351ba7"},
     {file = "isort-5.10.1.tar.gz", hash = "sha256:e8443a5e7a020e9d7f97f1d7d9cd17c88bcb3bc7e218bf9cf5095fe550be2951"},
 ]
+jedi = [
+    {file = "jedi-0.18.1-py2.py3-none-any.whl", hash = "sha256:637c9635fcf47945ceb91cd7f320234a7be540ded6f3e99a50cb6febdfd1ba8d"},
+    {file = "jedi-0.18.1.tar.gz", hash = "sha256:74137626a64a99c8eb6ae5832d99b3bdd7d29a3850fe2aa80a4126b2a7d949ab"},
+]
 jeepney = [
     {file = "jeepney-0.7.1-py3-none-any.whl", hash = "sha256:1b5a0ea5c0e7b166b2f5895b91a08c14de8915afda4407fb5022a195224958ac"},
     {file = "jeepney-0.7.1.tar.gz", hash = "sha256:fa9e232dfa0c498bd0b8a3a73b8d8a31978304dcef0515adc859d4e096f96f4f"},
@@ -1382,6 +1645,10 @@ keyring = [
     {file = "keyring-23.5.0-py3-none-any.whl", hash = "sha256:b0d28928ac3ec8e42ef4cc227822647a19f1d544f21f96457965dc01cf555261"},
     {file = "keyring-23.5.0.tar.gz", hash = "sha256:9012508e141a80bd1c0b6778d5c610dd9f8c464d75ac6774248500503f972fb9"},
 ]
+matplotlib-inline = [
+    {file = "matplotlib-inline-0.1.3.tar.gz", hash = "sha256:a04bfba22e0d1395479f866853ec1ee28eea1485c1d69a6faf00dc3e24ff34ee"},
+    {file = "matplotlib_inline-0.1.3-py3-none-any.whl", hash = "sha256:aed605ba3b72462d64d475a21a9296f400a19c4f74a31b59103d2a99ffd5aa5c"},
+]
 mccabe = [
     {file = "mccabe-0.6.1-py2.py3-none-any.whl", hash = "sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42"},
     {file = "mccabe-0.6.1.tar.gz", hash = "sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f"},
@@ -1484,10 +1751,22 @@ packaging = [
     {file = "packaging-21.3-py3-none-any.whl", hash = "sha256:ef103e05f519cdc783ae24ea4e2e0f508a9c99b2d4969652eed6a2e1ea5bd522"},
     {file = "packaging-21.3.tar.gz", hash = "sha256:dd47c42927d89ab911e606518907cc2d3a1f38bbd026385970643f9c5b8ecfeb"},
 ]
+parso = [
+    {file = "parso-0.8.3-py2.py3-none-any.whl", hash = "sha256:c001d4636cd3aecdaf33cbb40aebb59b094be2a74c556778ef5576c175e19e75"},
+    {file = "parso-0.8.3.tar.gz", hash = "sha256:8c07be290bb59f03588915921e29e8a50002acaf2cdc5fa0e0114f91709fafa0"},
+]
 pathspec = [
     {file = "pathspec-0.9.0-py2.py3-none-any.whl", hash = "sha256:7d15c4ddb0b5c802d161efc417ec1a2558ea2653c2e8ad9c19098201dc1c993a"},
     {file = "pathspec-0.9.0.tar.gz", hash = "sha256:e564499435a2673d586f6b2130bb5b95f04a3ba06f81b8f895b651a3c76aabb1"},
 ]
+pexpect = [
+    {file = "pexpect-4.8.0-py2.py3-none-any.whl", hash = "sha256:0b48a55dcb3c05f3329815901ea4fc1537514d6ba867a152b581d69ae3710937"},
+    {file = "pexpect-4.8.0.tar.gz", hash = "sha256:fc65a43959d153d0114afe13997d439c22823a27cefceb5ff35c2178c6784c0c"},
+]
+pickleshare = [
+    {file = "pickleshare-0.7.5-py2.py3-none-any.whl", hash = "sha256:9649af414d74d4df115d5d718f82acb59c9d418196b7b4290ed47a12ce62df56"},
+    {file = "pickleshare-0.7.5.tar.gz", hash = "sha256:87683d47965c1da65cdacaf31c8441d12b8044cdec9aca500cd78fc2c683afca"},
+]
 pillow = [
     {file = "Pillow-9.0.1-1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a5d24e1d674dd9d72c66ad3ea9131322819ff86250b30dc5821cbafcfa0b96b4"},
     {file = "Pillow-9.0.1-1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:2632d0f846b7c7600edf53c48f8f9f1e13e62f66a6dbc15191029d950bfed976"},
@@ -1541,6 +1820,18 @@ pre-commit = [
     {file = "pre_commit-2.17.0-py2.py3-none-any.whl", hash = "sha256:725fa7459782d7bec5ead072810e47351de01709be838c2ce1726b9591dad616"},
     {file = "pre_commit-2.17.0.tar.gz", hash = "sha256:c1a8040ff15ad3d648c70cc3e55b93e4d2d5b687320955505587fd79bbaed06a"},
 ]
+prompt-toolkit = [
+    {file = "prompt_toolkit-3.0.28-py3-none-any.whl", hash = "sha256:30129d870dcb0b3b6a53efdc9d0a83ea96162ffd28ffe077e94215b233dc670c"},
+    {file = "prompt_toolkit-3.0.28.tar.gz", hash = "sha256:9f1cd16b1e86c2968f2519d7fb31dd9d669916f515612c269d14e9ed52b51650"},
+]
+ptyprocess = [
+    {file = "ptyprocess-0.7.0-py2.py3-none-any.whl", hash = "sha256:4b41f3967fce3af57cc7e94b888626c18bf37a083e3651ca8feeb66d492fef35"},
+    {file = "ptyprocess-0.7.0.tar.gz", hash = "sha256:5c5d0a3b48ceee0b48485e0c26037c0acd7d29765ca3fbb5cb3831d347423220"},
+]
+pure-eval = [
+    {file = "pure_eval-0.2.2-py3-none-any.whl", hash = "sha256:01eaab343580944bc56080ebe0a674b39ec44a945e6d09ba7db3cb8cec289350"},
+    {file = "pure_eval-0.2.2.tar.gz", hash = "sha256:2b45320af6dfaa1750f543d714b6d1c520a1688dec6fd24d339063ce0aaa9ac3"},
+]
 py = [
     {file = "py-1.11.0-py2.py3-none-any.whl", hash = "sha256:607c53218732647dff4acdfcd50cb62615cedf612e72d1724fb1a0cc6405b378"},
     {file = "py-1.11.0.tar.gz", hash = "sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719"},
@@ -1705,6 +1996,10 @@ snowballstemmer = [
     {file = "snowballstemmer-2.2.0-py2.py3-none-any.whl", hash = "sha256:c8e1716e83cc398ae16824e5572ae04e0d9fc2c6b985fb0f900f5f0c96ecba1a"},
     {file = "snowballstemmer-2.2.0.tar.gz", hash = "sha256:09b16deb8547d3412ad7b590689584cd0fe25ec8db3be37788be3810cbf19cb1"},
 ]
+stack-data = [
+    {file = "stack_data-0.2.0-py3-none-any.whl", hash = "sha256:999762f9c3132308789affa03e9271bbbe947bf78311851f4d485d8402ed858e"},
+    {file = "stack_data-0.2.0.tar.gz", hash = "sha256:45692d41bd633a9503a5195552df22b583caf16f0b27c4e58c98d88c8b648e12"},
+]
 toml = [
     {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"},
     {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"},
@@ -1725,6 +2020,10 @@ tqdm = [
     {file = "tqdm-4.62.3-py2.py3-none-any.whl", hash = "sha256:8dd278a422499cd6b727e6ae4061c40b48fce8b76d1ccbf5d34fca9b7f925b0c"},
     {file = "tqdm-4.62.3.tar.gz", hash = "sha256:d359de7217506c9851b7869f3708d8ee53ed70a1b8edbba4dbcb47442592920d"},
 ]
+traitlets = [
+    {file = "traitlets-5.1.1-py3-none-any.whl", hash = "sha256:2d313cc50a42cd6c277e7d7dc8d4d7fedd06a2c215f78766ae7b1a66277e0033"},
+    {file = "traitlets-5.1.1.tar.gz", hash = "sha256:059f456c5a7c1c82b98c2e8c799f39c9b8128f6d0d46941ee118daace9eb70c7"},
+]
 twine = [
     {file = "twine-3.8.0-py3-none-any.whl", hash = "sha256:d0550fca9dc19f3d5e8eadfce0c227294df0a2a951251a4385797c8a6198b7c8"},
     {file = "twine-3.8.0.tar.gz", hash = "sha256:8efa52658e0ae770686a13b675569328f1fba9837e5de1867bfe5f46a9aefe19"},
@@ -1733,6 +2032,14 @@ typer = [
     {file = "typer-0.4.0-py3-none-any.whl", hash = "sha256:d81169725140423d072df464cad1ff25ee154ef381aaf5b8225352ea187ca338"},
     {file = "typer-0.4.0.tar.gz", hash = "sha256:63c3aeab0549750ffe40da79a1b524f60e08a2cbc3126c520ebf2eeaf507f5dd"},
 ]
+types-cryptography = [
+    {file = "types-cryptography-3.3.18.tar.gz", hash = "sha256:448feaf9ae31226149bc6bce1cf8fff54da661ef04837f7c0c316829885c3628"},
+    {file = "types_cryptography-3.3.18-py3-none-any.whl", hash = "sha256:d745cba4b1d0ec0a141b8b7693f887a02c332d8f6036c100db3cac98443e2ddb"},
+]
+types-pytz = [
+    {file = "types-pytz-2021.3.5.tar.gz", hash = "sha256:fef8de238ee95135952229a2a23bfb87bd63d5a6c8598106a46cfcf48f069ea8"},
+    {file = "types_pytz-2021.3.5-py3-none-any.whl", hash = "sha256:8831f689379ac9e2a62668157381379ed74b3702980e08e71f8673c179c4e3c7"},
+]
 typing-extensions = [
     {file = "typing_extensions-4.1.1-py3-none-any.whl", hash = "sha256:21c85e0fe4b9a155d0799430b0ad741cdce7e359660ccbd8b530613e8df88ce2"},
     {file = "typing_extensions-4.1.1.tar.gz", hash = "sha256:1a9462dcc3347a79b1f1c0271fbe79e844580bb598bafa1ed208b94da3cdcd42"},
@@ -1753,6 +2060,10 @@ virtualenv = [
     {file = "virtualenv-20.13.1-py2.py3-none-any.whl", hash = "sha256:45e1d053cad4cd453181ae877c4ffc053546ae99e7dd049b9ff1d9be7491abf7"},
     {file = "virtualenv-20.13.1.tar.gz", hash = "sha256:e0621bcbf4160e4e1030f05065c8834b4e93f4fcc223255db2a823440aca9c14"},
 ]
+wcwidth = [
+    {file = "wcwidth-0.2.5-py2.py3-none-any.whl", hash = "sha256:beb4802a9cebb9144e99086eff703a642a13d6a0052920003a230f3294bbe784"},
+    {file = "wcwidth-0.2.5.tar.gz", hash = "sha256:c4d647b99872929fdb7bdcaa4fbe7f01413ed3d98077df798530e5b04f116c83"},
+]
 webencodings = [
     {file = "webencodings-0.5.1-py2.py3-none-any.whl", hash = "sha256:a0af1213f3c2226497a97e2b3aa01a7e4bee4f403f95be16fc9acd2947514a78"},
     {file = "webencodings-0.5.1.tar.gz", hash = "sha256:b36a1c245f2d304965eb4e0a82848379241dc04b865afcc4aab16748587e1923"},

pyproject.toml

@@ -1,7 +1,7 @@
 [tool]
 [tool.poetry]
 name = "unifi-protect-backup"
-version = "0.4.0"
+version = "0.7.0"
 homepage = "https://github.com/ep1cman/unifi-protect-backup"
 description = "Python tool to backup unifi event clips in realtime."
 authors = ["sebastian.goscik <sebastian@goscik.com>"]
@@ -41,6 +41,9 @@ bump2version = {version = "^1.0.1", optional = true}
 tox-asdf = {version = "^0.1.0", optional = true}
 pyunifiprotect = "^3.2.1"
 aiocron = "^1.8"
+ipdb = {version = "^0.13.9", optional = true}
+types-pytz = {version = "^2021.3.5", optional = true}
+types-cryptography = {version = "^3.3.18", optional = true}
 
 [tool.poetry.extras]
 test = [
@@ -50,10 +53,12 @@ test = [
     "mypy",
     "flake8",
     "flake8-docstrings",
-    "pytest-cov"
+    "pytest-cov",
+    "types-pytz",
+    "types-cryptography"
     ]
 
-dev = ["tox", "pre-commit", "virtualenv", "pip", "twine", "toml", "bump2version", "tox-asdf"]
+dev = ["tox", "pre-commit", "virtualenv", "pip", "twine", "toml", "bump2version", "tox-asdf", "ipdb"]
 
 [tool.poetry.scripts]
 unifi-protect-backup = 'unifi_protect_backup.cli:main'

unifi_protect_backup/__init__.py

@@ -2,6 +2,6 @@
 
 __author__ = """sebastian.goscik"""
 __email__ = 'sebastian@goscik.com'
-__version__ = '0.4.0'
+__version__ = '0.7.0'
 
 from .unifi_protect_backup import UnifiProtectBackup

unifi_protect_backup/cli.py

@@ -6,16 +6,31 @@ import click
 
 from unifi_protect_backup import UnifiProtectBackup, __version__
 
+DETECTION_TYPES = ["motion", "person", "vehicle", "ring"]
+
+
+def _parse_detection_types(ctx, param, value):
+    # split columns by ',' and remove whitespace
+    types = [t.strip() for t in value.split(',')]
+
+    # validate passed columns
+    for t in types:
+        if t not in DETECTION_TYPES:
+            raise click.BadOptionUsage("detection-types", f"`{t}` is not an available detection type.", ctx)
+
+    return types
+
 
 @click.command()
 @click.version_option(__version__)
 @click.option('--address', required=True, envvar='UFP_ADDRESS', help='Address of Unifi Protect instance')
-@click.option('--port', default=443, envvar='UFP_PORT', help='Port of Unifi Protect instance')
+@click.option('--port', default=443, envvar='UFP_PORT', show_default=True, help='Port of Unifi Protect instance')
 @click.option('--username', required=True, envvar='UFP_USERNAME', help='Username to login to Unifi Protect instance')
 @click.option('--password', required=True, envvar='UFP_PASSWORD', help='Password for Unifi Protect user')
 @click.option(
     '--verify-ssl/--no-verify-ssl',
     default=True,
+    show_default=True,
     envvar='UFP_SSL_VERIFY',
     help="Set if you do not have a valid HTTPS Certificate for your instance",
 )
@@ -29,6 +44,7 @@ from unifi_protect_backup import UnifiProtectBackup, __version__
 @click.option(
     '--retention',
     default='7d',
+    show_default=True,
     envvar='RCLONE_RETENTION',
     help="How long should event clips be backed up for. Format as per the `--max-age` argument of "
     "`rclone` (https://rclone.org/filtering/#max-age-don-t-transfer-any-file-older-than-this)",
@@ -40,6 +56,15 @@ from unifi_protect_backup import UnifiProtectBackup, __version__
     help="Optional extra arguments to pass to `rclone rcat` directly. Common usage for this would "
     "be to set a bandwidth limit, for example.",
 )
+@click.option(
+    '--detection-types',
+    envvar='DETECTION_TYPES',
+    default=','.join(DETECTION_TYPES),
+    show_default=True,
+    help="A comma separated list of which types of detections to backup. "
+    f"Valid options are: {', '.join([f'`{t}`' for t in DETECTION_TYPES])}",
+    callback=_parse_detection_types,
+)
 @click.option(
     '--ignore-camera',
     'ignore_cameras',
@@ -48,6 +73,14 @@ from unifi_protect_backup import UnifiProtectBackup, __version__
     help="IDs of cameras for which events should not be backed up. Use multiple times to ignore "
     "multiple IDs. If being set as an environment variable the IDs should be separated by whitespace.",
 )
+@click.option(
+    '--file-structure-format',
+    envvar='FILE_STRUCTURE_FORMAT',
+    default="{camera_name}/{event.start:%Y-%m-%d}/{event.end:%Y-%m-%dT%H-%M-%S} {detection_type}.mp4",
+    show_default=True,
+    help="A Python format string used to generate the file structure/name on the rclone remote."
+    "For details of the fields available, see the projects `README.md` file.",
+)
 @click.option(
     '-v',
     '--verbose',

unifi_protect_backup/unifi_protect_backup.py

@@ -1,12 +1,16 @@
 """Main module."""
 import asyncio
+import json
 import logging
 import pathlib
+import re
 import shutil
 from asyncio.exceptions import TimeoutError
+from datetime import datetime, timedelta, timezone
 from typing import Callable, List, Optional
 
 import aiocron
+import pytz
 from aiohttp.client_exceptions import ClientPayloadError
 from pyunifiprotect import NvrError, ProtectApiClient
 from pyunifiprotect.data.nvr import Event
@@ -16,7 +20,7 @@ from pyunifiprotect.data.websocket import WSAction, WSSubscriptionMessage
 logger = logging.getLogger(__name__)
 
 
-class RcloneException(Exception):
+class SubprocessException(Exception):
     """Exception class for when rclone does not exit with `0`."""
 
     def __init__(self, stdout, stderr, returncode):
@@ -172,8 +176,11 @@ class UnifiProtectBackup:
         rclone_args (str): Extra args passed directly to `rclone rcat`.
         ignore_cameras (List[str]): List of camera IDs for which to not backup events
         verbose (int): How verbose to setup logging, see :func:`setup_logging` for details.
+        detection_types (List[str]): List of which detection types to backup.
+        file_structure_format (str): A Python format string for output file path
         _download_queue (asyncio.Queue): Queue of events that need to be backed up
         _unsub (Callable): Unsubscribe from the websocket callback
+        _has_ffprobe (bool): If ffprobe was found on the host
     """
 
     def __init__(
@@ -185,7 +192,9 @@ class UnifiProtectBackup:
         rclone_destination: str,
         retention: str,
         rclone_args: str,
+        detection_types: List[str],
         ignore_cameras: List[str],
+        file_structure_format: str,
         verbose: int,
         port: int = 443,
     ):
@@ -205,7 +214,9 @@ class UnifiProtectBackup:
                             (https://rclone.org/filtering/#max-age-don-t-transfer-any-file-older-than-this)
             rclone_args (str): A bandwidth limit which is passed to the `--bwlimit` argument of
                                    `rclone` (https://rclone.org/docs/#bwlimit-bandwidth-spec)
-            ignore_cameras (List[str]): List of camera IDs for which to not backup events
+            detection_types (List[str]): List of which detection types to backup.
+            ignore_cameras (List[str]): List of camera IDs for which to not backup events.
+            file_structure_format (str): A Python format string for output file path.
             verbose (int): How verbose to setup logging, see :func:`setup_logging` for details.
         """
         setup_logging(verbose)
@@ -225,10 +236,13 @@ class UnifiProtectBackup:
         logger.debug(f"  {rclone_args=}")
         logger.debug(f"  {ignore_cameras=}")
         logger.debug(f"  {verbose=}")
+        logger.debug(f"  {detection_types=}")
+        logger.debug(f"  {file_structure_format=}")
 
         self.rclone_destination = rclone_destination
         self.retention = retention
         self.rclone_args = rclone_args
+        self.file_structure_format = file_structure_format
 
         self.address = address
         self.port = port
@@ -247,6 +261,9 @@ class UnifiProtectBackup:
         self.ignore_cameras = ignore_cameras
         self._download_queue: asyncio.Queue = asyncio.Queue()
         self._unsub: Callable[[], None]
+        self.detection_types = detection_types
+
+        self._has_ffprobe = False
 
     async def start(self):
         """Bootstrap the backup process and kick off the main loop.
@@ -256,10 +273,16 @@ class UnifiProtectBackup:
         """
         logger.info("Starting...")
 
-        # Ensure rclone is installed and properly configured
+        # Ensure `rclone` is installed and properly configured
         logger.info("Checking rclone configuration...")
         await self._check_rclone()
 
+        # Check if `ffprobe` is available
+        ffprobe = shutil.which('ffprobe')
+        if ffprobe is not None:
+            logger.debug(f"ffprobe found: {ffprobe}")
+            self._has_ffprobe = True
+
         # Start the pyunifiprotect connection by calling `update`
         logger.info("Connecting to Unifi Protect...")
         await self._protect.update()
@@ -278,8 +301,8 @@ class UnifiProtectBackup:
         @aiocron.crontab("0 0 * * *")
         async def rclone_purge_old():
             logger.info("Deleting old files...")
-            cmd = f"rclone delete -vv --min-age {self.retention} '{self.rclone_destination}'"
-            cmd += f" && rclone rmdirs -vv --leave-root '{self.rclone_destination}'"
+            cmd = f'rclone delete -vv --min-age {self.retention} "{self.rclone_destination}"'
+            cmd += f' && rclone rmdirs -vv --leave-root "{self.rclone_destination}"'
             proc = await asyncio.create_subprocess_shell(
                 cmd,
                 stdout=asyncio.subprocess.PIPE,
@@ -361,14 +384,14 @@ class UnifiProtectBackup:
         """Check if rclone is installed and the specified remote is configured.
 
         Raises:
-            RcloneException: If rclone is not installed or it failed to list remotes
+            SubprocessException: If rclone is not installed or it failed to list remotes
             ValueError: The given rclone destination is for a remote that is not configured
 
         """
         rclone = shutil.which('rclone')
-        logger.debug(f"rclone found: {rclone}")
         if not rclone:
             raise RuntimeError("`rclone` is not installed on this system")
+        logger.debug(f"rclone found: {rclone}")
 
         cmd = "rclone listremotes -vv"
         proc = await asyncio.create_subprocess_shell(
@@ -380,7 +403,7 @@ class UnifiProtectBackup:
         logger.extra_debug(f"stdout:\n{stdout.decode()}")  # type: ignore
         logger.extra_debug(f"stderr:\n{stderr.decode()}")  # type: ignore
         if proc.returncode != 0:
-            raise RcloneException(stdout.decode(), stderr.decode(), proc.returncode)
+            raise SubprocessException(stdout.decode(), stderr.decode(), proc.returncode)
 
         # Check if the destination is for a configured remote
         for line in stdout.splitlines():
@@ -408,8 +431,22 @@ class UnifiProtectBackup:
             return
         if msg.new_obj.end is None:
             return
-        if msg.new_obj.type not in {EventType.MOTION, EventType.SMART_DETECT}:
+        if msg.new_obj.type not in [EventType.MOTION, EventType.SMART_DETECT, EventType.RING]:
             return
+        if msg.new_obj.type is EventType.MOTION and "motion" not in self.detection_types:
+            logger.extra_debug(f"Skipping unwanted motion detection event: {msg.new_obj.id}")  # type: ignore
+            return
+        if msg.new_obj.type is EventType.RING and "ring" not in self.detection_types:
+            logger.extra_debug(f"Skipping unwanted ring event: {msg.new_obj.id}")  # type: ignore
+            return
+        elif msg.new_obj.type is EventType.SMART_DETECT:
+            for event_smart_detection_type in msg.new_obj.smart_detect_types:
+                if event_smart_detection_type not in self.detection_types:
+                    logger.extra_debug(  # type: ignore
+                        f"Skipping unwanted {event_smart_detection_type} detection event: {msg.new_obj.id}"
+                    )
+                    return
+
         self._download_queue.put_nowait(msg.new_obj)
         logger.debug(f"Adding event {msg.new_obj.id} to queue (Current queue={self._download_queue.qsize()})")
 
@@ -425,13 +462,35 @@ class UnifiProtectBackup:
             try:
                 event = await self._download_queue.get()
 
+                # Fix timezones since pyunifiprotect sets all timestamps to UTC. Instead localize them to
+                # the timezone of the unifi protect NVR.
+                event.start = event.start.replace(tzinfo=pytz.utc).astimezone(self._protect.bootstrap.nvr.timezone)
+                event.end = event.end.replace(tzinfo=pytz.utc).astimezone(self._protect.bootstrap.nvr.timezone)
+
                 logger.info(f"Backing up event: {event.id}")
                 logger.debug(f"Remaining Queue: {self._download_queue.qsize()}")
                 logger.debug(f"  Camera: {await self._get_camera_name(event.camera_id)}")
-                logger.debug(f"  Type: {event.type}")
-                logger.debug(f"  Start: {event.start.strftime('%Y-%m-%dT%H-%M-%S')}")
-                logger.debug(f"  End: {event.end.strftime('%Y-%m-%dT%H-%M-%S')}")
-                logger.debug(f"  Duration: {event.end-event.start}")
+                if event.type == EventType.SMART_DETECT:
+                    logger.debug(f"  Type: {event.type} ({', '.join(event.smart_detect_types)})")
+                else:
+                    logger.debug(f"  Type: {event.type}")
+                logger.debug(f"  Start: {event.start.strftime('%Y-%m-%dT%H-%M-%S')} ({event.start.timestamp()})")
+                logger.debug(f"  End: {event.end.strftime('%Y-%m-%dT%H-%M-%S')} ({event.end.timestamp()})")
+                duration = (event.end - event.start).total_seconds()
+                logger.debug(f"  Duration: {duration}")
+
+                # Unifi protect does not return full video clips if the clip is requested too soon.
+                # There are two issues at play here:
+                #  - Protect will only cut a clip on an keyframe which happen every 5s
+                #  - Protect's pipeline needs a finite amount of time to make a clip available
+                # So we will wait 1.5x the keyframe interval to ensure that there is always ample video
+                # stored and Protect can return a full clip (which should be at least the length requested,
+                # but often longer)
+                time_since_event_ended = datetime.utcnow().replace(tzinfo=timezone.utc) - event.end
+                sleep_time = (timedelta(seconds=5 * 1.5) - time_since_event_ended).total_seconds()
+                if sleep_time > 0:
+                    logger.debug(f"  Sleeping ({sleep_time}s) to ensure clip is ready to download...")
+                    await asyncio.sleep(sleep_time)
 
                 # Download video
                 logger.debug("  Downloading video...")
@@ -450,6 +509,23 @@ class UnifiProtectBackup:
 
                 destination = await self.generate_file_path(event)
 
+                # Get the actual length of the downloaded video using ffprobe
+                if self._has_ffprobe:
+                    try:
+                        downloaded_duration = await self._get_video_length(video)
+                        msg = (
+                            f"  Downloaded video length: {downloaded_duration:.3f}s"
+                            f"({downloaded_duration - duration:+.3f}s)"
+                        )
+                        if downloaded_duration < duration:
+                            logger.warning(msg)
+                        else:
+                            logger.debug(msg)
+                    except SubprocessException as e:
+                        logger.warn("    `ffprobe` failed")
+                        logger.exception(e)
+
+                # Upload video
                 logger.debug("  Uploading video via rclone...")
                 logger.debug(f"    To: {destination}")
                 logger.debug(f"    Size: {human_readable_size(len(video))}")
@@ -457,7 +533,7 @@ class UnifiProtectBackup:
                     try:
                         await self._upload_video(video, destination, self.rclone_args)
                         break
-                    except RcloneException as e:
+                    except SubprocessException as e:
                         logger.warn(f"    Failed upload attempt {x+1}, retying in 1s")
                         logger.exception(e)
                         await asyncio.sleep(1)
@@ -485,7 +561,7 @@ class UnifiProtectBackup:
         Raises:
             RuntimeError: If rclone returns a non-zero exit code
         """
-        cmd = f"rclone rcat -vv {rclone_args} '{destination}'"
+        cmd = f'rclone rcat -vv {rclone_args} "{destination}"'
         proc = await asyncio.create_subprocess_shell(
             cmd,
             stdin=asyncio.subprocess.PIPE,
@@ -497,17 +573,39 @@ class UnifiProtectBackup:
             logger.extra_debug(f"stdout:\n{stdout.decode()}")  # type: ignore
             logger.extra_debug(f"stderr:\n{stderr.decode()}")  # type: ignore
         else:
-            raise RcloneException(stdout.decode(), stderr.decode(), proc.returncode)
+            raise SubprocessException(stdout.decode(), stderr.decode(), proc.returncode)
+
+    async def _get_video_length(self, video: bytes) -> float:
+        cmd = 'ffprobe -v quiet -show_streams -select_streams v:0 -of json -'
+        proc = await asyncio.create_subprocess_shell(
+            cmd,
+            stdin=asyncio.subprocess.PIPE,
+            stdout=asyncio.subprocess.PIPE,
+            stderr=asyncio.subprocess.PIPE,
+        )
+        stdout, stderr = await proc.communicate(video)
+        if proc.returncode == 0:
+            logger.extra_debug(f"stdout:\n{stdout.decode()}")  # type: ignore
+            logger.extra_debug(f"stderr:\n{stderr.decode()}")  # type: ignore
+
+            json_data = json.loads(stdout.decode())
+            return float(json_data['streams'][0]['duration'])
+
+        else:
+            raise SubprocessException(stdout.decode(), stderr.decode(), proc.returncode)
 
     async def generate_file_path(self, event: Event) -> pathlib.Path:
         """Generates the rclone destination path for the provided event.
 
-        Generates paths in the following structure:
-        ::
-          rclone_destination
-          |- Camera Name
-             |- {Date}
-                 |- {start timestamp} {event type} ({detections}).mp4
+        Generates rclone destination path for the given even based upon the format string
+        in `self.file_structure_format`.
+
+        Provides the following fields to the format string:
+          event: The `Event` object as per
+                 https://github.com/briis/pyunifiprotect/blob/master/pyunifiprotect/data/nvr.py
+          duration_seconds: The duration of the event in seconds
+          detection_type: A nicely formatted list of the event detection type and the smart detection types (if any)
+          camera_name: The name of the camera that generated this event
 
         Args:
             event: The event for which to create an output path
@@ -516,21 +614,23 @@ class UnifiProtectBackup:
             pathlib.Path: The rclone path the event should be backed up to
 
         """
-        path = pathlib.Path(self.rclone_destination)
         assert isinstance(event.camera_id, str)
-        path /= await self._get_camera_name(event.camera_id)  # directory per camera
-        path /= event.start.strftime("%Y-%m-%d")  # Directory per day
+        assert isinstance(event.start, datetime)
+        assert isinstance(event.end, datetime)
 
-        file_name = f"{event.start.strftime('%Y-%m-%dT%H-%M-%S')} {event.type}"
+        format_context = {
+            "event": event,
+            "duration_seconds": (event.end - event.start).total_seconds(),
+            "detection_type": f"{event.type} ({' '.join(event.smart_detect_types)})"
+            if event.smart_detect_types
+            else f"{event.type}",
+            "camera_name": await self._get_camera_name(event.camera_id),
+        }
 
-        if event.smart_detect_types:
-            detections = " ".join(event.smart_detect_types)
-            file_name += f" ({detections})"
-        file_name += ".mp4"
+        file_path = self.file_structure_format.format(**format_context)
+        file_path = re.sub(r'[^\w\-_\.\(\)/ ]', '', file_path)  # Sanitize any invalid chars
 
-        path /= file_name
-
-        return path
+        return pathlib.Path(f"{self.rclone_destination}/{file_path}")
 
     async def _get_camera_name(self, id: str):
         try: