Compare commits

...

7 Commits

Author SHA1 Message Date
Dobby
ef0cf38f83 Fix ruff formatting error 2025-07-27 15:04:53 +01:00
Dobby
2bd48014a0 Split retention and missing range 2025-07-27 15:04:53 +01:00
dependabot[bot]
afe025be1d Bump aiohttp from 3.11.16 to 3.12.14 in the pip group across 1 directory
---
updated-dependencies:
- dependency-name: aiohttp
  dependency-version: 3.12.14
  dependency-type: direct:production
  dependency-group: pip
...

Signed-off-by: dependabot[bot] <support@github.com>
2025-07-27 11:22:56 +01:00
Dobby
a14ff1bf30 Fix skipping overlapping events on chunk boundary 2025-07-27 11:13:27 +01:00
Dobby
ba64722937 Fix cp command always running 2025-07-27 11:11:54 +01:00
Dobby
65d8e66e79 Fix slow missing events check 2025-07-27 11:10:53 +01:00
Dobby
cb54078153 Move log to after delete finished 2025-07-27 11:08:59 +01:00
7 changed files with 50 additions and 17 deletions

View File

@@ -23,7 +23,7 @@ retention period.
## Features
- Listens to events in real-time via the Unifi Protect websocket API
- Ensures any previous and/or missed events within the retention period are also backed up
- Ensures any previous and/or missed events within the missing range are also backed up
- Supports uploading to a [wide range of storage systems using `rclone`](https://rclone.org/overview/)
- Automatic pruning of old clips
@@ -123,6 +123,10 @@ Options:
`--max-age` argument of `rclone`
(https://rclone.org/filtering/#max-age-don-t-transfer-any-file-
older-than-this) [default: 7d]
--missing-range TEXT How far back should missing events be checked for. Defaults to
the same as the retention time. Format as per the `--max-age`
argument of `rclone` (https://rclone.org/filtering/#max-age-don-
t-transfer-any-file-older-than-this)
--rclone-args TEXT Optional extra arguments to pass to `rclone rcat` directly.
Common usage for this would be to set a bandwidth limit, for
example.
@@ -131,14 +135,21 @@ Options:
instead of using the recycle bin on a destination. Google Drive
example: `--drive-use-trash=false`
--detection-types TEXT A comma separated list of which types of detections to backup.
Valid options are: `motion`, `person`, `vehicle`, `ring`
[default: motion,person,vehicle,ring]
Valid options are: `motion`, `ring`, `line`, `fingerprint`,
`nfc`, `person`, `animal`, `vehicle`, `licensePlate`, `package`,
`face`, `car`, `pet`, `alrmSmoke`, `alrmCmonx`, `smoke_cmonx`,
`alrmSiren`, `alrmBabyCry`, `alrmSpeak`, `alrmBark`,
`alrmBurglar`, `alrmCarHorn`, `alrmGlassBreak` [default: motion
,ring,line,fingerprint,nfc,person,animal,vehicle,licensePlate,pa
ckage,face,car,pet,alrmSmoke,alrmCmonx,smoke_cmonx,alrmSiren,alr
mBabyCry,alrmSpeak,alrmBark,alrmBurglar,alrmCarHorn,alrmGlassBre
ak]
--ignore-camera TEXT IDs of cameras for which events should not be backed up. Use
multiple times to ignore multiple IDs. If being set as an
environment variable the IDs should be separated by whitespace.
Alternatively, use a Unifi user with a role which has access
restricted to the subset of cameras that you wish to backup.
--camera TEXT IDs of *ONLY* cameras for which events should be backed up. Use
--camera TEXT IDs of *ONLY* cameras for which events should be backed up. Use
multiple times to include multiple IDs. If being set as an
environment variable the IDs should be separated by whitespace.
Alternatively, use a Unifi user with a role which has access
@@ -214,6 +225,7 @@ always take priority over environment variables):
- `UFP_PORT`
- `UFP_SSL_VERIFY`
- `RCLONE_RETENTION`
- `MISSING_RANGE`
- `RCLONE_DESTINATION`
- `RCLONE_ARGS`
- `RCLONE_PURGE_ARGS`

View File

@@ -4,7 +4,7 @@ mkdir -p /config/rclone
# For backwards compatibility
[[ -f "/root/.config/rclone/rclone.conf" ]] && \
echo "DEPRECATED: Copying rclone conf from /root/.config/rclone/rclone.conf, please change your mount to /config/rclone/rclone.conf"
echo "DEPRECATED: Copying rclone conf from /root/.config/rclone/rclone.conf, please change your mount to /config/rclone/rclone.conf" && \
cp \
/root/.config/rclone/rclone.conf \
/config/rclone/rclone.conf

View File

@@ -30,7 +30,7 @@ dependencies = [
"async-lru>=2.0.4",
"aiolimiter>=1.1.0",
"uiprotect==7.14.1",
"aiohttp==3.11.16",
"aiohttp==3.12.14",
]
[project.urls]

View File

@@ -30,8 +30,11 @@ def _parse_detection_types(ctx, param, value):
return types
def parse_rclone_retention(ctx, param, retention) -> relativedelta:
def parse_rclone_retention(ctx, param, retention) -> relativedelta | None:
"""Parse the rclone `retention` parameter into a relativedelta which can then be used to calculate datetimes."""
if retention is None:
return None
matches = {k: int(v) for v, k in re.findall(r"([\d]+)(ms|s|m|h|d|w|M|y)", retention)}
# Check that we matched the whole string
@@ -79,6 +82,15 @@ def parse_rclone_retention(ctx, param, retention) -> relativedelta:
"(https://rclone.org/filtering/#max-age-don-t-transfer-any-file-older-than-this)",
callback=parse_rclone_retention,
)
@click.option(
"--missing-range",
default=None,
envvar="MISSING_RANGE",
help="How far back should missing events be checked for. Defaults to the same as the retention time. "
"Format as per the `--max-age` argument of `rclone` "
"(https://rclone.org/filtering/#max-age-don-t-transfer-any-file-older-than-this)",
callback=parse_rclone_retention,
)
@click.option(
"--rclone-args",
default="",
@@ -261,6 +273,9 @@ def main(**kwargs):
)
raise SystemExit(200) # throw 200 = arg error, service will not be restarted (docker)
if kwargs.get("missing_range") is None:
kwargs["missing_range"] = kwargs.get("retention")
# Only create the event listener and run if validation passes
event_listener = UnifiProtectBackup(**kwargs)
run(event_listener.start(), stop_on_unhandled_errors=True)

View File

@@ -83,8 +83,8 @@ class MissingEventChecker:
if not unifi_events:
break # No completed events to process
# Next chunks start time should be the end of the oldest complete event in the current chunk
start_time = max([event.end for event in unifi_events.values() if event.end is not None])
# Next chunks start time should be the start of the oldest complete event in the current chunk
start_time = max([event.start for event in unifi_events.values() if event.end is not None])
# Get list of events that have been backed up from the database
@@ -105,10 +105,9 @@ class MissingEventChecker:
if current_upload is not None:
uploading_event_ids.add(current_upload.id)
existing_ids = db_event_ids | downloading_event_ids | uploading_event_ids
missing_events = {
event_id: event
for event_id, event in unifi_events.items()
if event_id not in (db_event_ids | downloading_event_ids | uploading_event_ids)
event_id: event for event_id, event in unifi_events.items() if event_id not in existing_ids
}
# Exclude events of unwanted types

View File

@@ -70,8 +70,8 @@ class Purge:
# For every backup for this event
async with self._db.execute(f"SELECT * FROM backups WHERE id = '{event_id}'") as backup_cursor:
async for _, remote, file_path in backup_cursor:
logger.debug(f" Deleted: {remote}:{file_path}")
await delete_file(f"{remote}:{file_path}", self.rclone_purge_args)
logger.debug(f" Deleted: {remote}:{file_path}")
deleted_a_file = True
# delete event from database

View File

@@ -68,6 +68,7 @@ class UnifiProtectBackup:
verify_ssl: bool,
rclone_destination: str,
retention: relativedelta,
missing_range: relativedelta,
rclone_args: str,
rclone_purge_args: str,
detection_types: List[str],
@@ -98,9 +99,13 @@ class UnifiProtectBackup:
rclone_destination (str): `rclone` destination path in the format
{rclone remote}:{path on remote}. E.g.
`gdrive:/backups/unifi_protect`
retention (str): How long should event clips be backed up for. Format as per the
retention (relativedelta): How long should event clips be backed up for. Format as per the
`--max-age` argument of `rclone`
(https://rclone.org/filtering/#max-age-don-t-transfer-any-file-older-than-this)
missing_range (relativedelta): How far back should missing events be checked for. Defaults to
the same as the retention time. Format as per the
`--max-age` argument of `rclone`
(https://rclone.org/filtering/#max-age-don-t-transfer-any-file-older-than-this)
rclone_args (str): A bandwidth limit which is passed to the `--bwlimit` argument of
`rclone` (https://rclone.org/docs/#bwlimit-bandwidth-spec)
rclone_purge_args (str): Optional extra arguments to pass to `rclone delete` directly.
@@ -144,6 +149,7 @@ class UnifiProtectBackup:
logger.debug(f" {verify_ssl=}")
logger.debug(f" {rclone_destination=}")
logger.debug(f" {retention=}")
logger.debug(f" {missing_range=}")
logger.debug(f" {rclone_args=}")
logger.debug(f" {rclone_purge_args=}")
logger.debug(f" {ignore_cameras=}")
@@ -163,6 +169,7 @@ class UnifiProtectBackup:
self.rclone_destination = rclone_destination
self.retention = retention
self.missing_range = missing_range
self.rclone_args = rclone_args
self.rclone_purge_args = rclone_purge_args
self.file_structure_format = file_structure_format
@@ -314,15 +321,15 @@ class UnifiProtectBackup:
tasks.append(purge.start())
# Create missing event task
# This will check all the events within the retention period, if any have been missed and not backed up
# they will be added to the event queue
# This will check all the events within the missing_range period, if any have been missed and not
# backed up. they will be added to the event queue
missing = MissingEventChecker(
self._protect,
self._db,
download_queue,
downloader,
uploaders,
self.retention,
self.missing_range,
self.detection_types,
self.ignore_cameras,
self.cameras,