Properly handle rate limits in the scanner

scanner/scanner/__init__.py

@@ -1,5 +1,5 @@
 import logging
-from asyncio import CancelledError, TaskGroup, create_task
+from asyncio import CancelledError, TaskGroup, create_task, sleep
 from contextlib import asynccontextmanager
 
 from fastapi import FastAPI
@@ -48,12 +48,16 @@ async def background_startup(
 	processor: RequestProcessor,
 	is_master: bool | None,
 ):
+	async def scan():
+		# wait for everything to startup & resume before scanning
+		await sleep(30)
+		await scanner.scan(remove_deleted=True)
+
 	async with TaskGroup() as tg:
 		_ = tg.create_task(processor.listen(tg))
 		if is_master:
 			_ = tg.create_task(scanner.monitor())
-			_ = tg.create_task(scanner.scan(remove_deleted=True))
-
+			_ = tg.create_task(scan())
 
 async def cancel():
 	raise CancelledError()

scanner/scanner/providers/themoviedatabase.py

@@ -7,7 +7,7 @@ from statistics import mean
 from types import TracebackType
 from typing import Any, cast, override
 
-from aiohttp import ClientSession
+from aiohttp import ClientResponseError, ClientSession
 from langcodes import Language
 
 from ..models.collection import Collection, CollectionTranslation
@@ -643,7 +643,21 @@ class TheMovieDatabase(Provider):
 		async with self._client.get(path, params=params) as r:
 			if not_found_fail and r.status == 404:
 				raise ProviderError(not_found_fail)
-			r.raise_for_status()
+			if r.status == 429:
+				retry_after = r.headers.get("Retry-After")
+				delay = float(retry_after) if retry_after else 2.0
+				await asyncio.sleep(delay)
+				return await self._get(
+					path, params=params, not_found_fail=not_found_fail
+				)
+			if r.status >= 400:
+				raise ClientResponseError(
+					r.request_info,
+					r.history,
+					status=r.status,
+					message=await r.text(),
+					headers=r.headers,
+				)
 			return await r.json()
 
 	def _map_genres(self, genres: Generator[int]) -> list[Genre]: