add timed cache decorator

This commit is contained in:
Dmitry Afanasyev 2022-08-27 18:22:57 +03:00
parent cd307e6403
commit 4c49b58546
8 changed files with 52 additions and 30 deletions

View File

@ -4,7 +4,7 @@ from aiogram import Bot, types
from aiogram.contrib.middlewares.logging import LoggingMiddleware from aiogram.contrib.middlewares.logging import LoggingMiddleware
from aiogram.dispatcher import Dispatcher from aiogram.dispatcher import Dispatcher
from aiogram.utils.callback_data import CallbackData from aiogram.utils.callback_data import CallbackData
from app.core.parse_web import get_driver, get_ttl_hash, parse_site from app.core.parse_web import get_driver, parse_site
from app.settings import TELEGRAM_API_TOKEN from app.settings import TELEGRAM_API_TOKEN
bot = Bot(token=TELEGRAM_API_TOKEN) bot = Bot(token=TELEGRAM_API_TOKEN)
@ -36,7 +36,7 @@ def get_keyboard() -> types.InlineKeyboardMarkup:
async def home_office( async def home_office(
query: types.CallbackQuery, callback_data: dict[str, str] query: types.CallbackQuery, callback_data: dict[str, str]
) -> types.Message: ) -> types.Message:
driver = get_driver(ttl_hash=get_ttl_hash(seconds=10)) driver = get_driver()
text = parse_site( text = parse_site(
driver=driver, driver=driver,
url='https://yandex.ru/maps/213/moscow/stops/stop__9640740/' url='https://yandex.ru/maps/213/moscow/stops/stop__9640740/'
@ -53,7 +53,7 @@ async def home_office(
async def office_home( async def office_home(
query: types.CallbackQuery, callback_data: dict[str, str] query: types.CallbackQuery, callback_data: dict[str, str]
) -> types.Message: ) -> types.Message:
driver = get_driver(ttl_hash=get_ttl_hash()) driver = get_driver()
text = parse_site( text = parse_site(
driver=driver, driver=driver,
url='https://yandex.ru/maps/213/moscow/stops/stop__9640288/?' url='https://yandex.ru/maps/213/moscow/stops/stop__9640288/?'
@ -79,7 +79,7 @@ async def echo(message: types.Message) -> types.Message:
async def morning_bus_mailing(chat_ids: list[int]) -> None: async def morning_bus_mailing(chat_ids: list[int]) -> None:
driver = get_driver(ttl_hash=get_ttl_hash()) driver = get_driver()
text = parse_site( text = parse_site(
driver=driver, driver=driver,
url='https://yandex.ru/maps/213/moscow/stops/stop__9640740/' url='https://yandex.ru/maps/213/moscow/stops/stop__9640740/'

View File

@ -1,11 +0,0 @@
import sys
from loguru import logger
logger.remove()
logger.add(
sink=sys.stdout,
colorize=True,
level='DEBUG',
format="<cyan>{time:DD.MM.YYYY HH:mm:ss}</cyan> | <level>{level}</level> | <magenta>{message}</magenta>",
)

View File

@ -1,12 +1,11 @@
import os import os
import tarfile import tarfile
import time import time
from functools import lru_cache
from pathlib import Path from pathlib import Path
import wget import wget
from app.core.logger import logger from app.core.utils import logger, timed_cache
from app.settings import BASE_DIR, GECKO_DRIVER_VERSION from app.settings import BASE_DIR, DRIVER_SESSION_TTL, GECKO_DRIVER_VERSION
from selenium import webdriver from selenium import webdriver
from selenium.common.exceptions import ( from selenium.common.exceptions import (
NoSuchElementException, NoSuchElementException,
@ -93,14 +92,8 @@ def parse_site(url: str, message: str, driver: RemoteWebDriver | None = None) ->
return answer return answer
def get_ttl_hash(seconds: int = 28) -> int: @timed_cache(seconds=DRIVER_SESSION_TTL)
"""Return the same value withing `seconds` time period""" def get_driver() -> RemoteWebDriver:
return round(time.time() / seconds)
@lru_cache
def get_driver(ttl_hash: int | None = None) -> RemoteWebDriver:
del ttl_hash
opt = options.Options() opt = options.Options()
opt.headless = True opt.headless = True
driver = RemoteWebDriver( driver = RemoteWebDriver(

35
app/core/utils.py Normal file
View File

@ -0,0 +1,35 @@
import sys
from datetime import datetime, timedelta
from functools import lru_cache, wraps
from typing import Any
from loguru import logger
logger.remove()
logger.add(
sink=sys.stdout,
colorize=True,
level='DEBUG',
format="<cyan>{time:DD.MM.YYYY HH:mm:ss}</cyan> | <level>{level}</level> | <magenta>{message}</magenta>",
)
def timed_cache(**timedelta_kwargs: Any) -> Any:
def _wrapper(func: Any) -> Any:
update_delta = timedelta(**timedelta_kwargs)
next_update = datetime.utcnow() + update_delta
# Apply @lru_cache to f with no cache size limit
cached_func = lru_cache(None)(func)
@wraps(func)
def _wrapped(*args: Any, **kwargs: Any) -> Any:
nonlocal next_update
now = datetime.utcnow()
if now >= next_update:
cached_func.cache_clear()
next_update = now + update_delta
return cached_func(*args, **kwargs)
return _wrapped
return _wrapper

View File

@ -11,8 +11,8 @@ from aiohttp import web
sys.path.append(str(Path(__file__).parent.parent)) sys.path.append(str(Path(__file__).parent.parent))
from app.core.bot import bot, dispatcher from app.core.bot import bot, dispatcher
from app.core.logger import logger
from app.core.scheduler import asyncio_schedule from app.core.scheduler import asyncio_schedule
from app.core.utils import logger
from app.settings import ( from app.settings import (
START_WITH_WEBHOOK, START_WITH_WEBHOOK,
TELEGRAM_API_TOKEN, TELEGRAM_API_TOKEN,
@ -88,7 +88,7 @@ async def get_updates_from_queue() -> None:
async def create_app() -> web.Application: async def create_app() -> web.Application:
application = web.Application() application = web.Application()
application.router.add_get('/', health_check) application.router.add_get(f'{WEBHOOK_PATH}/', health_check)
application.router.add_post( application.router.add_post(
f'{WEBHOOK_PATH}/{TELEGRAM_API_TOKEN}', put_updates_on_queue f'{WEBHOOK_PATH}/{TELEGRAM_API_TOKEN}', put_updates_on_queue
) )

View File

@ -29,3 +29,5 @@ WEBAPP_HOST = config('WEBAPP_HOST', default='127.0.0.1') # or ip
WEBAPP_PORT = config('WEBAPP_PORT', cast=int, default=8084) WEBAPP_PORT = config('WEBAPP_PORT', cast=int, default=8084)
START_WITH_WEBHOOK = config('START_WITH_WEBHOOK', cast=bool, default=False) START_WITH_WEBHOOK = config('START_WITH_WEBHOOK', cast=bool, default=False)
DRIVER_SESSION_TTL = 28 # driver cache ttl session in seconds

View File

@ -4,7 +4,7 @@
# Removing some headers for improved security: # Removing some headers for improved security:
header -Server header -Server
route /transport/{$TELEGRAM_API_TOKEN} { route /transport/* {
reverse_proxy transport_bot:8084 reverse_proxy transport_bot:8084
} }
} }

View File

@ -24,13 +24,16 @@ services:
args: args:
USER: web USER: web
restart: unless-stopped restart: unless-stopped
environment:
- SESSION_TIMED_OUT=30s
networks: networks:
transport_bot_network: transport_bot_network:
ipv4_address: 200.20.0.10 ipv4_address: 200.20.0.10
volumes: volumes:
- ./deploy/browsers.json:/etc/selenoid/browsers.json:ro - ./deploy/browsers.json:/etc/selenoid/browsers.json:ro
- /var/run/docker.sock:/var/run/docker.sock - /var/run/docker.sock:/var/run/docker.sock
command: ["-conf", "/etc/selenoid/browsers.json", "-limit", "10", "-container-network", "transport_bot_network"] command: ["-conf", "/etc/selenoid/browsers.json", "-limit", "10",
"-container-network", "transport_bot_network", "-timeout", "30s"]
expose: expose:
- "4444" - "4444"