add queue to bot

app/core/bot.py

@@ -1,26 +1,21 @@
 import asyncio
+from concurrent.futures import ThreadPoolExecutor
 
 from aiogram import Bot, types
 from aiogram.contrib.middlewares.logging import LoggingMiddleware
 from aiogram.dispatcher import Dispatcher
 from aiogram.utils.callback_data import CallbackData
-from app.core.logger import logger
+from app.core.parse_web import get_driver, parse_site
-from app.core.parse_web import (
-    configure_firefox_driver,
-    download_gecko_driver,
-    parse_site,
-)
 from app.settings import API_TOKEN
 
 bot = Bot(token=API_TOKEN)
 dispatcher = Dispatcher(bot)
 dispatcher.middleware.setup(LoggingMiddleware())
 
-download_gecko_driver()
-driver = configure_firefox_driver()
-
 stations_cb = CallbackData('station', 'direction')
 
+executor = ThreadPoolExecutor(10)
+
 
 def get_keyboard() -> types.InlineKeyboardMarkup:
     """
@@ -44,12 +39,16 @@ async def home_office(
     query: types.CallbackQuery, callback_data: dict[str, str]
 ) -> types.Message:
 
-    text = parse_site(
+    driver = get_driver()
-        driver=driver,
+
-        url='https://yandex.ru/maps/213/moscow/stops/stop__9640740/'
+    url = (
-        '?l=masstransit&ll=37.527754%2C55.823507&tab=overview&z=21',
+        'https://yandex.ru/maps/213/moscow/stops/stop__9640740/'
-        message='Остановка Б. Академическая ул, д. 15',
+        '?l=masstransit&ll=37.527754%2C55.823507&tab=overview&z=21'
     )
+    message = 'Остановка Б. Академическая ул, д. 15'
+
+    loop = asyncio.get_running_loop()
+    text = await loop.run_in_executor(executor, parse_site, url, message, driver)
 
     return await bot.send_message(
         query.message.chat.id, text, reply_markup=get_keyboard()
@@ -61,12 +60,16 @@ async def office_home(
     query: types.CallbackQuery, callback_data: dict[str, str]
 ) -> types.Message:
 
-    text = parse_site(
+    driver = get_driver()
-        driver=driver,
+
-        url='https://yandex.ru/maps/213/moscow/stops/stop__9640288/?'
+    url = (
-        'l=masstransit&ll=37.505338%2C55.800160&tab=overview&z=211',
+        'https://yandex.ru/maps/213/moscow/stops/stop__9640288/'
-        message='Остановка Улица Алабяна',
+        '?l=masstransit&ll=37.505338%2C55.800160&tab=overview&z=211'
     )
+    message = 'Остановка Улица Алабяна'
+
+    loop = asyncio.get_running_loop()
+    text = await loop.run_in_executor(executor, parse_site, url, message, driver)
 
     return await bot.send_message(
         query.message.chat.id, text, reply_markup=get_keyboard()
@@ -86,12 +89,16 @@ async def echo(message: types.Message) -> types.Message:
 
 
 async def morning_bus_mailing(chat_ids: list[int]) -> None:
-    text = parse_site(
+    driver = get_driver()
-        driver=driver,
+
-        url='https://yandex.ru/maps/213/moscow/stops/stop__9640740/'
+    url = (
-        '?l=masstransit&ll=37.527754%2C55.823507&tab=overview&z=21',
+        'https://yandex.ru/maps/213/moscow/stops/stop__9640740/'
-        message='Остановка Б. Академическая ул, д. 15',
+        '?l=masstransit&ll=37.527754%2C55.823507&tab=overview&z=21'
     )
+    message = 'Остановка Б. Академическая ул, д. 15'
+
+    loop = asyncio.get_running_loop()
+    text = await loop.run_in_executor(executor, parse_site, url, message, driver)
     await asyncio.gather(
         *[
             bot.send_message(

app/core/parse_web.py

@@ -7,10 +7,14 @@ import wget
 from app.core.logger import logger
 from app.settings import BASE_DIR, GECKO_DRIVER_VERSION
 from selenium import webdriver
-from selenium.common.exceptions import NoSuchElementException, WebDriverException
+from selenium.common.exceptions import (
+    NoSuchElementException,
+    StaleElementReferenceException,
+    WebDriverException,
+)
 from selenium.webdriver.firefox import options
 from selenium.webdriver.firefox.service import Service
-from selenium.webdriver.firefox.webdriver import WebDriver
+from selenium.webdriver.firefox.webdriver import RemoteWebDriver, WebDriver
 
 
 def download_gecko_driver() -> None:
@@ -52,7 +56,7 @@ def parse_site(url: str, message: str, driver: WebDriver | None = None) -> str:
         logger.error('Driver is not configured')
         return 'Что-то пошло не так. :( Драйвер Firefox не сконфигурирован.'
     driver.get(url)
-    time.sleep(1)
+    time.sleep(5)
     elements = driver.find_elements(
         by='class name', value='masstransit-vehicle-snippet-view'
     )
@@ -68,9 +72,6 @@ def parse_site(url: str, message: str, driver: WebDriver | None = None) -> str:
             bus_300_arrival = element.find_element(
                 by='class name', value='masstransit-prognoses-view__title-text'
             )
-        except NoSuchElementException:
-            pass
-        try:
             bus_t19 = element.find_element(
                 by='css selector', value='[aria-label="т19"]'
             )
@@ -79,6 +80,9 @@ def parse_site(url: str, message: str, driver: WebDriver | None = None) -> str:
             )
         except NoSuchElementException:
             pass
+        except StaleElementReferenceException:
+            pass
+
     answer = f'{message}\n\n'
     if not all([bus_300, bus_t19]) or not all([bus_300_arrival, bus_t19_arrival]):
         return 'Автобусов 300 или Т19 не найдено. \n\nСмотри на карте :)'
@@ -86,4 +90,14 @@ def parse_site(url: str, message: str, driver: WebDriver | None = None) -> str:
         answer += f'Автобус {bus_300.text} - {bus_300_arrival.text}\n'
     if bus_t19 and bus_t19_arrival:
         answer += f'Автобус {bus_t19.text} - {bus_t19_arrival.text}'
+    print(answer)
     return answer
+
+
+def get_driver() -> RemoteWebDriver:
+    opt = options.Options()
+    opt.headless = True
+    driver = RemoteWebDriver(
+        command_executor='http://main-server.lan:4444/wd/hub', options=opt
+    )
+    return driver