From aa75ec6f97e34e03018bf9ec1440ef3edcd89935 Mon Sep 17 00:00:00 2001 From: Radhakrishnan Sethuraman Date: Mon, 18 Nov 2024 17:27:29 -0600 Subject: [PATCH 01/15] Updates for rclone parallel uploads --- unifi_protect_backup/cli.py | 7 ++ unifi_protect_backup/missing_event_checker.py | 6 +- .../unifi_protect_backup_core.py | 2 + unifi_protect_backup/uploader.py | 67 +++++++++++-------- 4 files changed, 52 insertions(+), 30 deletions(-) diff --git a/unifi_protect_backup/cli.py b/unifi_protect_backup/cli.py index 1cdba14..063ca83 100644 --- a/unifi_protect_backup/cli.py +++ b/unifi_protect_backup/cli.py @@ -226,6 +226,13 @@ what the web UI does. This might be more stable if you are experiencing a lot of failed downloads with the default downloader. """, ) +@click.option( + "--rclone-parallel-uploads", + default=1, + show_default=True, + envvar="RCLONE_PARALLEL_UPLOADS", + help="Number of parallel uploads", +) def main(**kwargs): """A Python based tool for backing up Unifi Protect event clips as they occur.""" event_listener = UnifiProtectBackup(**kwargs) diff --git a/unifi_protect_backup/missing_event_checker.py b/unifi_protect_backup/missing_event_checker.py index 6c245f4..fb0829f 100644 --- a/unifi_protect_backup/missing_event_checker.py +++ b/unifi_protect_backup/missing_event_checker.py @@ -97,9 +97,9 @@ class MissingEventChecker: downloading_event_ids.add(current_download.id) uploading_event_ids = {event.id for event, video in self._uploader.upload_queue._queue} # type: ignore - current_upload = self._uploader.current_event - if current_upload is not None: - uploading_event_ids.add(current_upload.id) + for current_upload in self._uploader.current_events: + if current_upload is not None: + uploading_event_ids.add(current_upload.id) missing_event_ids = set(unifi_events.keys()) - (db_event_ids | downloading_event_ids | uploading_event_ids) diff --git a/unifi_protect_backup/unifi_protect_backup_core.py b/unifi_protect_backup/unifi_protect_backup_core.py index 18715ba..b1a6179 100644 --- a/unifi_protect_backup/unifi_protect_backup_core.py +++ b/unifi_protect_backup/unifi_protect_backup_core.py @@ -76,6 +76,7 @@ class UnifiProtectBackup: download_rate_limit: float | None = None, port: int = 443, use_experimental_downloader: bool = False, + rclone_parallel_uploads: int = 1, ): """Will configure logging settings and the Unifi Protect API (but not actually connect). @@ -270,6 +271,7 @@ class UnifiProtectBackup: self.file_structure_format, self._db, self.color_logging, + self.rclone_parallel_uploads ) tasks.append(uploader.start()) diff --git a/unifi_protect_backup/uploader.py b/unifi_protect_backup/uploader.py index 2a860bc..b5df6fc 100644 --- a/unifi_protect_backup/uploader.py +++ b/unifi_protect_backup/uploader.py @@ -4,7 +4,8 @@ import logging import pathlib import re from datetime import datetime - +import os +import asyncio import aiosqlite from uiprotect import ProtectApiClient from uiprotect.data.nvr import Event @@ -34,6 +35,7 @@ class VideoUploader: file_structure_format: str, db: aiosqlite.Connection, color_logging: bool, + rclone_parallel_uploads: int ): """Init. @@ -53,11 +55,42 @@ class VideoUploader: self._file_structure_format: str = file_structure_format self._db: aiosqlite.Connection = db self.current_event = None + self.rclone_parallel_uploads = rclone_parallel_uploads self.base_logger = logging.getLogger(__name__) setup_event_logger(self.base_logger, color_logging) self.logger = logging.LoggerAdapter(self.base_logger, {"event": ""}) + async def _upload_worker(self, semaphore, worker_id): + async with semaphore: + while True: + try: + event, video = await self.upload_queue.get() + self.current_events[worker_id] = event + + logger = logging.LoggerAdapter(self.base_logger, {'event': f' [{event.id}]'}) + + logger.info(f"Uploading event: {event.id}") + logger.debug( + f" Remaining Upload Queue: {self.upload_queue.qsize_files()}" + f" ({human_readable_size(self.upload_queue.qsize())})" + ) + + destination = await self._generate_file_path(event) + logger.debug(f" Destination: {destination}") + + try: + await self._upload_video(video, destination, self._rclone_args) + await self._update_database(event, destination) + logger.debug("Uploaded") + except SubprocessException: + logger.error(f" Failed to upload file: '{destination}'") + + except Exception as e: + logger.error(f"Unexpected exception occurred, abandoning event {event.id}:", exc_info=e) + + self.current_events[worker_id] = None + async def start(self): """Main loop. @@ -65,33 +98,13 @@ class VideoUploader: using rclone, finally it updates the database """ self.logger.info("Starting Uploader") - while True: - try: - event, video = await self.upload_queue.get() - self.current_event = event + + rclone_transfers = self.rclone_parallel_uploads + self.current_events = [None] * rclone_transfers + semaphore = asyncio.Semaphore(rclone_transfers) - self.logger = logging.LoggerAdapter(self.base_logger, {"event": f" [{event.id}]"}) - - self.logger.info(f"Uploading event: {event.id}") - self.logger.debug( - f" Remaining Upload Queue: {self.upload_queue.qsize_files()}" - f" ({human_readable_size(self.upload_queue.qsize())})" - ) - - destination = await self._generate_file_path(event) - self.logger.debug(f" Destination: {destination}") - - try: - await self._upload_video(video, destination, self._rclone_args) - await self._update_database(event, destination) - self.logger.debug("Uploaded") - except SubprocessException: - self.logger.error(f" Failed to upload file: '{destination}'") - - self.current_event = None - - except Exception as e: - self.logger.error(f"Unexpected exception occurred, abandoning event {event.id}:", exc_info=e) + workers = [self._upload_worker(semaphore, i) for i in range(rclone_transfers)] + await asyncio.gather(*workers) async def _upload_video(self, video: bytes, destination: pathlib.Path, rclone_args: str): """Upload video using rclone. From 61370be1c2797849b8db06adecc3bff6878f3674 Mon Sep 17 00:00:00 2001 From: Radhakrishnan Sethuraman Date: Mon, 18 Nov 2024 19:47:13 -0600 Subject: [PATCH 02/15] remove other os options --- .github/workflows/dev.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml index 7b135fc..b203ace 100644 --- a/.github/workflows/dev.yml +++ b/.github/workflows/dev.yml @@ -19,7 +19,7 @@ jobs: strategy: matrix: python-versions: [3.9] - os: [ubuntu-18.04, macos-latest, windows-latest] + os: [ubuntu-18.04] runs-on: ${{ matrix.os }} # Steps represent a sequence of tasks that will be executed as part of the job From d41d2ef08374b1044158f426841fc031877ec1f2 Mon Sep 17 00:00:00 2001 From: Radhakrishnan Sethuraman Date: Mon, 18 Nov 2024 19:49:26 -0600 Subject: [PATCH 03/15] fix setup-python version --- .github/workflows/dev.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml index b203ace..3eb2ac5 100644 --- a/.github/workflows/dev.yml +++ b/.github/workflows/dev.yml @@ -57,7 +57,7 @@ jobs: # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 + - uses: actions/setup-python@v5 with: python-version: 3.10 From df62f671e115c4501fb8b5ca3e0458832c300fc1 Mon Sep 17 00:00:00 2001 From: Radhakrishnan Sethuraman Date: Mon, 18 Nov 2024 19:50:31 -0600 Subject: [PATCH 04/15] update setup-python@v5 --- .github/workflows/dev.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml index 3eb2ac5..caf66f6 100644 --- a/.github/workflows/dev.yml +++ b/.github/workflows/dev.yml @@ -26,7 +26,7 @@ jobs: steps: # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 + - uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-versions }} From 01a90b899d3f7f2efbd1c783fec8ced1b65c6f83 Mon Sep 17 00:00:00 2001 From: Radhakrishnan Sethuraman Date: Mon, 18 Nov 2024 19:56:07 -0600 Subject: [PATCH 05/15] fix python versions --- .github/workflows/dev.yml | 29 ++--------------------------- 1 file changed, 2 insertions(+), 27 deletions(-) diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml index caf66f6..06590c9 100644 --- a/.github/workflows/dev.yml +++ b/.github/workflows/dev.yml @@ -18,35 +18,10 @@ jobs: # The type of runner that the job will run on strategy: matrix: - python-versions: [3.9] + python-versions: [3.13] os: [ubuntu-18.04] runs-on: ${{ matrix.os }} - # Steps represent a sequence of tasks that will be executed as part of the job - steps: - # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - - uses: actions/checkout@v2 - - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-versions }} - - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install poetry tox tox-gh-actions - - - name: test with tox - run: - tox - - - name: list files - run: ls -l . - - - uses: codecov/codecov-action@v1 - with: - fail_ci_if_error: true - files: coverage.xml - dev_container: name: Create dev container runs-on: ubuntu-20.04 @@ -59,7 +34,7 @@ jobs: - uses: actions/setup-python@v5 with: - python-version: 3.10 + python-version: 3.13 - name: Install dependencies run: | From 9f0ab3030fb528761ad9aceef36687df21db1056 Mon Sep 17 00:00:00 2001 From: Radhakrishnan Sethuraman Date: Mon, 18 Nov 2024 19:57:22 -0600 Subject: [PATCH 06/15] update dev --- .github/workflows/dev.yml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml index 06590c9..fd1a2e0 100644 --- a/.github/workflows/dev.yml +++ b/.github/workflows/dev.yml @@ -14,13 +14,6 @@ on: # A workflow run is made up of one or more jobs that can run sequentially or in parallel jobs: # This workflow contains a single job called "test" - test: - # The type of runner that the job will run on - strategy: - matrix: - python-versions: [3.13] - os: [ubuntu-18.04] - runs-on: ${{ matrix.os }} dev_container: name: Create dev container From 018063c413e2a472d7611eaad6de0d3e83e977ba Mon Sep 17 00:00:00 2001 From: Radhakrishnan Sethuraman Date: Mon, 18 Nov 2024 20:17:12 -0600 Subject: [PATCH 07/15] remove passing RCLONE_PARALLEL_UPLOADS --- unifi_protect_backup/cli.py | 1 + unifi_protect_backup/unifi_protect_backup_core.py | 6 ++---- unifi_protect_backup/uploader.py | 3 +-- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/unifi_protect_backup/cli.py b/unifi_protect_backup/cli.py index 063ca83..d7a8820 100644 --- a/unifi_protect_backup/cli.py +++ b/unifi_protect_backup/cli.py @@ -228,6 +228,7 @@ a lot of failed downloads with the default downloader. ) @click.option( "--rclone-parallel-uploads", + "rclone-parallel-uploads", default=1, show_default=True, envvar="RCLONE_PARALLEL_UPLOADS", diff --git a/unifi_protect_backup/unifi_protect_backup_core.py b/unifi_protect_backup/unifi_protect_backup_core.py index b1a6179..ac8adab 100644 --- a/unifi_protect_backup/unifi_protect_backup_core.py +++ b/unifi_protect_backup/unifi_protect_backup_core.py @@ -75,8 +75,7 @@ class UnifiProtectBackup: color_logging: bool = False, download_rate_limit: float | None = None, port: int = 443, - use_experimental_downloader: bool = False, - rclone_parallel_uploads: int = 1, + use_experimental_downloader: bool = False ): """Will configure logging settings and the Unifi Protect API (but not actually connect). @@ -270,8 +269,7 @@ class UnifiProtectBackup: self.rclone_args, self.file_structure_format, self._db, - self.color_logging, - self.rclone_parallel_uploads + self.color_logging ) tasks.append(uploader.start()) diff --git a/unifi_protect_backup/uploader.py b/unifi_protect_backup/uploader.py index b5df6fc..791e918 100644 --- a/unifi_protect_backup/uploader.py +++ b/unifi_protect_backup/uploader.py @@ -35,7 +35,6 @@ class VideoUploader: file_structure_format: str, db: aiosqlite.Connection, color_logging: bool, - rclone_parallel_uploads: int ): """Init. @@ -99,7 +98,7 @@ class VideoUploader: """ self.logger.info("Starting Uploader") - rclone_transfers = self.rclone_parallel_uploads + rclone_transfers = int(os.getenv('RCLONE_PARALLEL_UPLOADS', '1')) self.current_events = [None] * rclone_transfers semaphore = asyncio.Semaphore(rclone_transfers) From e79bbb206bb3812f7ac6168a8b03366b16753225 Mon Sep 17 00:00:00 2001 From: Radhakrishnan Sethuraman Date: Mon, 18 Nov 2024 20:38:07 -0600 Subject: [PATCH 08/15] remove env var --- unifi_protect_backup/cli.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/unifi_protect_backup/cli.py b/unifi_protect_backup/cli.py index d7a8820..1cdba14 100644 --- a/unifi_protect_backup/cli.py +++ b/unifi_protect_backup/cli.py @@ -226,14 +226,6 @@ what the web UI does. This might be more stable if you are experiencing a lot of failed downloads with the default downloader. """, ) -@click.option( - "--rclone-parallel-uploads", - "rclone-parallel-uploads", - default=1, - show_default=True, - envvar="RCLONE_PARALLEL_UPLOADS", - help="Number of parallel uploads", -) def main(**kwargs): """A Python based tool for backing up Unifi Protect event clips as they occur.""" event_listener = UnifiProtectBackup(**kwargs) From d9216a14f907392c82ed052cc3bcb478a12083d3 Mon Sep 17 00:00:00 2001 From: Radhakrishnan Sethuraman Date: Mon, 18 Nov 2024 20:53:34 -0600 Subject: [PATCH 09/15] clean up --- unifi_protect_backup/uploader.py | 1 - 1 file changed, 1 deletion(-) diff --git a/unifi_protect_backup/uploader.py b/unifi_protect_backup/uploader.py index 791e918..faeb8bd 100644 --- a/unifi_protect_backup/uploader.py +++ b/unifi_protect_backup/uploader.py @@ -54,7 +54,6 @@ class VideoUploader: self._file_structure_format: str = file_structure_format self._db: aiosqlite.Connection = db self.current_event = None - self.rclone_parallel_uploads = rclone_parallel_uploads self.base_logger = logging.getLogger(__name__) setup_event_logger(self.base_logger, color_logging) From 956ed7371466f0401dae199bd546e87e6c17b0d9 Mon Sep 17 00:00:00 2001 From: Radhakrishnan Sethuraman Date: Mon, 18 Nov 2024 21:01:13 -0600 Subject: [PATCH 10/15] fix variable name --- unifi_protect_backup/uploader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/unifi_protect_backup/uploader.py b/unifi_protect_backup/uploader.py index faeb8bd..7089576 100644 --- a/unifi_protect_backup/uploader.py +++ b/unifi_protect_backup/uploader.py @@ -53,7 +53,7 @@ class VideoUploader: self._rclone_args: str = rclone_args self._file_structure_format: str = file_structure_format self._db: aiosqlite.Connection = db - self.current_event = None + self.current_events = [] self.base_logger = logging.getLogger(__name__) setup_event_logger(self.base_logger, color_logging) From bab1d8f81d66f41231ab0af8a0c995ab07f8aa55 Mon Sep 17 00:00:00 2001 From: Radhakrishnan Sethuraman Date: Fri, 6 Dec 2024 10:50:26 -0600 Subject: [PATCH 11/15] fix file name --- unifi_protect_backup/uploader.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/unifi_protect_backup/uploader.py b/unifi_protect_backup/uploader.py index 7089576..4be3ef0 100644 --- a/unifi_protect_backup/uploader.py +++ b/unifi_protect_backup/uploader.py @@ -174,7 +174,7 @@ class VideoUploader: "camera_name": await get_camera_name(self._protect, event.camera_id), } - file_path = self._file_structure_format.format(**format_context) + file_path = self._file_structure_format.format(**format_context).lower() file_path = re.sub(r"[^\w\-_\.\(\)/ ]", "", file_path) # Sanitize any invalid chars - + file_path = file_path.replace(" ", "_") return pathlib.Path(f"{self._rclone_destination}/{file_path}") From 88bb5ba378b6eff526ae5fdec31b8ee2e5a26d83 Mon Sep 17 00:00:00 2001 From: Radhakrishnan Sethuraman Date: Mon, 9 Dec 2024 17:18:26 -0600 Subject: [PATCH 12/15] ignore events more than 10 mins --- unifi_protect_backup/downloader.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/unifi_protect_backup/downloader.py b/unifi_protect_backup/downloader.py index 47a5d43..b746ca6 100644 --- a/unifi_protect_backup/downloader.py +++ b/unifi_protect_backup/downloader.py @@ -180,13 +180,18 @@ class VideoDownloader: assert isinstance(event.camera_id, str) assert isinstance(event.start, datetime) assert isinstance(event.end, datetime) + diff = (event.end - event.start).total_seconds() + if diff > 600: + self.logger.info(f" Event exceeds 10 mins. Ignoring event {event.id}") + return None + try: video = await self._protect.get_camera_video(event.camera_id, event.start, event.end) assert isinstance(video, bytes) break except (AssertionError, ClientPayloadError, TimeoutError) as e: self.logger.warning(f" Failed download attempt {x+1}, retying in 1s", exc_info=e) - await asyncio.sleep(1) + await asyncio.sleep(0.5) else: self.logger.error(f"Download failed after 5 attempts, abandoning event {event.id}:") return None From 4edd936cb67c94a1872247b0ddc45f10e2e05ddf Mon Sep 17 00:00:00 2001 From: Radhakrishnan Sethuraman Date: Mon, 9 Dec 2024 17:31:47 -0600 Subject: [PATCH 13/15] reduce retries --- unifi_protect_backup/downloader.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/unifi_protect_backup/downloader.py b/unifi_protect_backup/downloader.py index b746ca6..5bd0bf9 100644 --- a/unifi_protect_backup/downloader.py +++ b/unifi_protect_backup/downloader.py @@ -175,8 +175,9 @@ class VideoDownloader: async def _download(self, event: Event) -> Optional[bytes]: """Downloads the video clip for the given event.""" - self.logger.debug(" Downloading video...") - for x in range(5): + + for x in range(3): + self.logger.debug(" Downloading video...") assert isinstance(event.camera_id, str) assert isinstance(event.start, datetime) assert isinstance(event.end, datetime) @@ -191,9 +192,9 @@ class VideoDownloader: break except (AssertionError, ClientPayloadError, TimeoutError) as e: self.logger.warning(f" Failed download attempt {x+1}, retying in 1s", exc_info=e) - await asyncio.sleep(0.5) + await asyncio.sleep(1) else: - self.logger.error(f"Download failed after 5 attempts, abandoning event {event.id}:") + self.logger.error(f"Download failed after 3 attempts, abandoning event {event.id}:") return None self.logger.debug(f" Downloaded video size: {human_readable_size(len(video))}s") From 8bbe8a08c2bd50f9b142daca68d83f552ec112d5 Mon Sep 17 00:00:00 2001 From: Radhakrishnan Sethuraman Date: Wed, 11 Dec 2024 20:44:18 -0600 Subject: [PATCH 14/15] update for ignoring events taking more than 60 seconds --- unifi_protect_backup/downloader.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/unifi_protect_backup/downloader.py b/unifi_protect_backup/downloader.py index 5bd0bf9..1c2bd7a 100644 --- a/unifi_protect_backup/downloader.py +++ b/unifi_protect_backup/downloader.py @@ -181,16 +181,17 @@ class VideoDownloader: assert isinstance(event.camera_id, str) assert isinstance(event.start, datetime) assert isinstance(event.end, datetime) - diff = (event.end - event.start).total_seconds() - if diff > 600: - self.logger.info(f" Event exceeds 10 mins. Ignoring event {event.id}") - return None + request_start_time = datetime.now() try: video = await self._protect.get_camera_video(event.camera_id, event.start, event.end) assert isinstance(video, bytes) break except (AssertionError, ClientPayloadError, TimeoutError) as e: + diff_seconds = (datetime.now() - request_start_time).total_seconds() + if diff_seconds > 60: + self.logger.error(f"Ignoring event. Total wait: {diff_seconds}. Camera: {await get_camera_name(self._protect, event.camera_id)}. Start: {event.start.strftime('%Y-%m-%dT%H-%M-%S')} ({event.start.timestamp()}) End: {event.end.strftime('%Y-%m-%dT%H-%M-%S')} ({event.end.timestamp()})", exc_info=e) + break self.logger.warning(f" Failed download attempt {x+1}, retying in 1s", exc_info=e) await asyncio.sleep(1) else: From 2c0afeaaa42244275c8bab8653b8be1522cf2137 Mon Sep 17 00:00:00 2001 From: Radhakrishnan Sethuraman Date: Fri, 13 Dec 2024 12:28:37 -0600 Subject: [PATCH 15/15] update for ignoring --- unifi_protect_backup/downloader.py | 5 +++-- unifi_protect_backup/purge.py | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/unifi_protect_backup/downloader.py b/unifi_protect_backup/downloader.py index 1c2bd7a..5635155 100644 --- a/unifi_protect_backup/downloader.py +++ b/unifi_protect_backup/downloader.py @@ -176,7 +176,7 @@ class VideoDownloader: async def _download(self, event: Event) -> Optional[bytes]: """Downloads the video clip for the given event.""" - for x in range(3): + for x in range(5): self.logger.debug(" Downloading video...") assert isinstance(event.camera_id, str) assert isinstance(event.start, datetime) @@ -191,11 +191,12 @@ class VideoDownloader: diff_seconds = (datetime.now() - request_start_time).total_seconds() if diff_seconds > 60: self.logger.error(f"Ignoring event. Total wait: {diff_seconds}. Camera: {await get_camera_name(self._protect, event.camera_id)}. Start: {event.start.strftime('%Y-%m-%dT%H-%M-%S')} ({event.start.timestamp()}) End: {event.end.strftime('%Y-%m-%dT%H-%M-%S')} ({event.end.timestamp()})", exc_info=e) + await self._ignore_event(event) break self.logger.warning(f" Failed download attempt {x+1}, retying in 1s", exc_info=e) await asyncio.sleep(1) else: - self.logger.error(f"Download failed after 3 attempts, abandoning event {event.id}:") + self.logger.error(f"Download failed after 5 attempts, abandoning event {event.id}:") return None self.logger.debug(f" Downloaded video size: {human_readable_size(len(video))}s") diff --git a/unifi_protect_backup/purge.py b/unifi_protect_backup/purge.py index feda6e1..0957e70 100644 --- a/unifi_protect_backup/purge.py +++ b/unifi_protect_backup/purge.py @@ -85,5 +85,5 @@ class Purge: logger.error("Unexpected exception occurred during purge:", exc_info=e) next_purge_time = datetime.now() + self.interval - logger.extra_debug(f"sleeping until {next_purge_time}") + logger.debug(f"sleeping until {next_purge_time}") await wait_until(next_purge_time)