jgaunt/unifi-protect-backup
1
0
Fork 0
mirror of https://github.com/ep1cman/unifi-protect-backup.git synced 2025-12-21 10:43:20 +00:00
Code Issues Packages Projects Releases Wiki Activity

Updates for rclone parallel uploads

Browse Source
This commit is contained in:
Radhakrishnan Sethuraman
2024-11-18 17:27:29 -06:00
parent c067dbd9f7
commit aa75ec6f97
4 changed files with 52 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files

67
unifi_protect_backup/uploader.py
View File

@@ -4,7 +4,8 @@ import logging
import pathlib
import re
from datetime import datetime
import os
import asyncio
import aiosqlite
from uiprotect import ProtectApiClient
from uiprotect.data.nvr import Event
@@ -34,6 +35,7 @@ class VideoUploader:
file_structure_format: str,
db: aiosqlite.Connection,
color_logging: bool,
rclone_parallel_uploads: int
):
"""Init.
@@ -53,11 +55,42 @@ class VideoUploader:
self._file_structure_format: str = file_structure_format
self._db: aiosqlite.Connection = db
self.current_event = None
self.rclone_parallel_uploads = rclone_parallel_uploads
self.base_logger = logging.getLogger(__name__)
setup_event_logger(self.base_logger, color_logging)
self.logger = logging.LoggerAdapter(self.base_logger, {"event": ""})
async def _upload_worker(self, semaphore, worker_id):
async with semaphore:
while True:
try:
event, video = await self.upload_queue.get()
self.current_events[worker_id] = event
logger = logging.LoggerAdapter(self.base_logger, {'event': f' [{event.id}]'})
logger.info(f"Uploading event: {event.id}")
logger.debug(
f" Remaining Upload Queue: {self.upload_queue.qsize_files()}"
f" ({human_readable_size(self.upload_queue.qsize())})"
)
destination = await self._generate_file_path(event)
logger.debug(f" Destination: {destination}")
try:
await self._upload_video(video, destination, self._rclone_args)
await self._update_database(event, destination)
logger.debug("Uploaded")
except SubprocessException:
logger.error(f" Failed to upload file: '{destination}'")
except Exception as e:
logger.error(f"Unexpected exception occurred, abandoning event {event.id}:", exc_info=e)
self.current_events[worker_id] = None
async def start(self):
"""Main loop.
@@ -65,33 +98,13 @@ class VideoUploader:
using rclone, finally it updates the database
"""
self.logger.info("Starting Uploader")
while True:
try:
event, video = await self.upload_queue.get()
self.current_event = event
rclone_transfers = self.rclone_parallel_uploads
self.current_events = [None] * rclone_transfers
semaphore = asyncio.Semaphore(rclone_transfers)
self.logger = logging.LoggerAdapter(self.base_logger, {"event": f" [{event.id}]"})
self.logger.info(f"Uploading event: {event.id}")
self.logger.debug(
f" Remaining Upload Queue: {self.upload_queue.qsize_files()}"
f" ({human_readable_size(self.upload_queue.qsize())})"
)
destination = await self._generate_file_path(event)
self.logger.debug(f" Destination: {destination}")
try:
await self._upload_video(video, destination, self._rclone_args)
await self._update_database(event, destination)
self.logger.debug("Uploaded")
except SubprocessException:
self.logger.error(f" Failed to upload file: '{destination}'")
self.current_event = None
except Exception as e:
self.logger.error(f"Unexpected exception occurred, abandoning event {event.id}:", exc_info=e)
workers = [self._upload_worker(semaphore, i) for i in range(rclone_transfers)]
await asyncio.gather(*workers)
async def _upload_video(self, video: bytes, destination: pathlib.Path, rclone_args: str):
"""Upload video using rclone.