From 031a7fade78c03ac29008b71c923ca54c50db3f6 Mon Sep 17 00:00:00 2001 From: Dmitry Afanasyev Date: Sat, 8 Apr 2023 14:49:46 +0300 Subject: [PATCH] run commands in ThreadPool --- app/config/.env.template | 19 +++++++----- app/core/bot.py | 66 ++++++++++++++++++++++------------------ app/core/parse_web.py | 9 +++--- app/core/scheduler.py | 6 ++-- app/settings.py | 4 ++- deploy/Caddyfile | 4 +-- docker-compose.yml | 4 ++- scripts/start-bot.sh | 19 ++++++++---- 8 files changed, 74 insertions(+), 57 deletions(-) diff --git a/app/config/.env.template b/app/config/.env.template index ee61ab0..219c053 100644 --- a/app/config/.env.template +++ b/app/config/.env.template @@ -1,14 +1,17 @@ -TELEGRAM_API_TOKEN= +TELEGRAM_API_TOKEN="123456789:AABBCCDDEEFFaabbccddeeff-1234567890" # webhook settings -WEBHOOK_HOST= -WEBHOOK_PATH= +WEBHOOK_HOST="https://mydomain.com" +WEBHOOK_PATH="/transport" # webserver settings -WEBAPP_HOST=127.0.0.1 -WEBAPP_PORT=8084 +WEBAPP_HOST="127.0.0.1" +WEBAPP_PORT="8080" -# set to 1 to start with webhook. Else bot will start on polling method -START_WITH_WEBHOOK= +# set to true to start with webhook. Else bot will start on polling method +START_WITH_WEBHOOK="true" -GECKO_DRIVER_VERSION=0.31.0 \ No newline at end of file +GECKO_DRIVER_VERSION="0.32.0" + +# chat ids for scheduler tasks +CHAT_IDS="123456789,987654321" \ No newline at end of file diff --git a/app/core/bot.py b/app/core/bot.py index f0bb5dc..377cb94 100644 --- a/app/core/bot.py +++ b/app/core/bot.py @@ -1,4 +1,5 @@ import asyncio +from concurrent.futures.thread import ThreadPoolExecutor from dataclasses import dataclass from aiogram import Bot, types @@ -9,6 +10,8 @@ from aiogram.utils.callback_data import CallbackData from app.core.parse_web import WebParser from app.settings import TELEGRAM_API_TOKEN +executor = ThreadPoolExecutor(10) + @dataclass class TransportBot: @@ -46,16 +49,14 @@ class TransportBot: async def home_office( query: types.CallbackQuery, callback_data: dict[str, str] ) -> types.Message: - driver = WebParser.get_driver() - text = WebParser.parse_yandex_maps( - driver=driver, - url='https://yandex.ru/maps/213/moscow/stops/stop__9640740/?ll=37.527924%2C55.823470&tab=overview&z=21', - message='Остановка Б. Академическая ул, д. 15', - buses=[ - '300', - 'т19', - ], - ) + url = 'https://yandex.ru/maps/213/moscow/stops/stop__9640740/?ll=37.527924%2C55.823470&tab=overview&z=21' + message = 'Остановка Б. Академическая ул, д. 15' + buses = [ + '300', + 'т19', + ] + + text = await TransportBot._get_buses_data(url=url, message=message, buses=buses) return await TransportBot.bot.send_message( query.message.chat.id, text, reply_markup=TransportBot.get_keyboard() @@ -66,16 +67,14 @@ class TransportBot: async def office_home( query: types.CallbackQuery, callback_data: dict[str, str] ) -> types.Message: - driver = WebParser.get_driver() - text = WebParser.parse_yandex_maps( - driver=driver, - url='https://yandex.ru/maps/213/moscow/stops/stop__9640288/?ll=37.505402%2C55.800214&tab=overview&z=21', - message='Остановка Улица Алабяна', - buses=[ - '300', - 'т19', - ], - ) + url = 'https://yandex.ru/maps/213/moscow/stops/stop__9640288/?ll=37.505402%2C55.800214&tab=overview&z=21' + message = 'Остановка Улица Алабяна' + buses = [ + '300', + 'т19', + ] + + text = await TransportBot._get_buses_data(url=url, message=message, buses=buses) return await TransportBot.bot.send_message( query.message.chat.id, text, reply_markup=TransportBot.get_keyboard() @@ -97,16 +96,15 @@ class TransportBot: if not chat_ids: return None - driver = WebParser.get_driver() - text = WebParser.parse_yandex_maps( - driver=driver, - url='https://yandex.ru/maps/213/moscow/stops/stop__9640740/?ll=37.527924%2C55.823470&tab=overview&z=21', - message='Остановка Б. Академическая ул, д. 15', - buses=[ - '300', - 'т19', - ], - ) + url = 'https://yandex.ru/maps/213/moscow/stops/stop__9640740/?ll=37.527924%2C55.823470&tab=overview&z=21' + message = 'Остановка Б. Академическая ул, д. 15' + buses = [ + '300', + 'т19', + ] + + text = await TransportBot._get_buses_data(url=url, message=message, buses=buses) + kwargs = {'reply_markup': TransportBot.get_keyboard()} if show_keyboard else {} await asyncio.gather( @@ -120,3 +118,11 @@ class TransportBot: for chat_id in chat_ids ] ) + + @staticmethod + async def _get_buses_data(url: str, message: str, buses: list[str]) -> str: + driver = WebParser.get_driver() + loop = asyncio.get_event_loop() + return await loop.run_in_executor( + executor, WebParser.parse_yandex_maps, url, message, buses, driver + ) diff --git a/app/core/parse_web.py b/app/core/parse_web.py index 9d743e2..ed87722 100644 --- a/app/core/parse_web.py +++ b/app/core/parse_web.py @@ -15,15 +15,14 @@ from app.settings import DRIVER_SESSION_TTL class WebParser: @staticmethod def parse_yandex_maps( - *, url: str, message: str, buses: list[str], - driver: WebDriver | None = None, + driver: WebDriver | None, ) -> str: if not driver: - logger.error('Driver is not configured') - return 'Что-то пошло не так. :( Драйвер Firefox не сконфигурирован.' + logger.error('Web driver is not configured') + return 'Что-то пошло не так. :( Веб драйвер не сконфигурирован.' driver.get(url) time.sleep(1) @@ -63,7 +62,7 @@ class WebParser: @staticmethod @timed_cache(seconds=DRIVER_SESSION_TTL) - def get_driver() -> WebDriver: + def get_driver() -> WebDriver | None: opt = webdriver.ChromeOptions() opt.add_argument('--headless') driver = webdriver.Remote( diff --git a/app/core/scheduler.py b/app/core/scheduler.py index 361021e..7e8f0fc 100644 --- a/app/core/scheduler.py +++ b/app/core/scheduler.py @@ -4,6 +4,7 @@ from apscheduler.schedulers.asyncio import AsyncIOScheduler from app.core.bot import TransportBot from app.core.utils import logger +from settings import CHAT_IDS bot_cron_jobs = { 'morning_home->work_bus': { @@ -39,10 +40,7 @@ bot_cron_jobs = { }, ], 'func_kwargs': { - 'chat_ids': [ - 417070387, # me - # 431571617, # Lenok - ] + 'chat_ids': CHAT_IDS, }, } } diff --git a/app/settings.py b/app/settings.py index a4a63e3..d49318e 100644 --- a/app/settings.py +++ b/app/settings.py @@ -1,6 +1,6 @@ from pathlib import Path -from decouple import AutoConfig +from decouple import AutoConfig, Csv # Build paths inside the project like this: BASE_DIR.joinpath('some') # `pathlib` is better than writing: dirname(dirname(dirname(__file__))) @@ -30,4 +30,6 @@ WEBAPP_PORT = config('WEBAPP_PORT', cast=int, default=8084) START_WITH_WEBHOOK = config('START_WITH_WEBHOOK', cast=bool, default=False) +CHAT_IDS = config('CHAT_IDS', cast=Csv(int), default=[]) # chat ids for scheduler tasks + DRIVER_SESSION_TTL = 28 # selenium driver session cache ttl in seconds diff --git a/deploy/Caddyfile b/deploy/Caddyfile index faaa223..d5c0a6f 100644 --- a/deploy/Caddyfile +++ b/deploy/Caddyfile @@ -4,8 +4,8 @@ # Removing some headers for improved security: header -Server - route /transport/* { - reverse_proxy transport_bot:8084 + route {$WEBHOOK_PATH}/* { + reverse_proxy transport_bot:8080 } } diff --git a/docker-compose.yml b/docker-compose.yml index 87517ec..07d8a3d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -45,6 +45,8 @@ services: args: USER: web restart: unless-stopped + env_file: + - app/config/.env depends_on: - selenoid volumes: @@ -53,7 +55,7 @@ services: transport_bot_network: ipv4_address: 200.20.0.11 expose: - - "8084" + - "8080" command: bash start-bot.sh diff --git a/scripts/start-bot.sh b/scripts/start-bot.sh index 80d298d..a5ab6e5 100644 --- a/scripts/start-bot.sh +++ b/scripts/start-bot.sh @@ -2,9 +2,16 @@ echo "starting the bot" -gunicorn main:create_app \ - --bind 0.0.0.0:8084 \ - --worker-class aiohttp.GunicornWebWorker \ - --timeout 150 \ - --max-requests 2000 \ - --max-requests-jitter 400 +if [[ "${START_WITH_WEBHOOK}" == "true" ]] +then + echo "Starting bot in webhook mode..." + gunicorn main:create_app \ + --bind ${WEBAPP_HOST}:${WEBAPP_PORT} \ + --worker-class aiohttp.GunicornWebWorker \ + --timeout 150 \ + --max-requests 2000 \ + --max-requests-jitter 400 +else + echo "Starting bot in polling mode..." + python main.py +fi