diff --git a/app/core/bot.py b/app/core/bot.py index f86fa54..a615485 100644 --- a/app/core/bot.py +++ b/app/core/bot.py @@ -1,26 +1,21 @@ import asyncio +from concurrent.futures import ThreadPoolExecutor from aiogram import Bot, types from aiogram.contrib.middlewares.logging import LoggingMiddleware from aiogram.dispatcher import Dispatcher from aiogram.utils.callback_data import CallbackData -from app.core.logger import logger -from app.core.parse_web import ( - configure_firefox_driver, - download_gecko_driver, - parse_site, -) +from app.core.parse_web import get_driver, parse_site from app.settings import API_TOKEN bot = Bot(token=API_TOKEN) dispatcher = Dispatcher(bot) dispatcher.middleware.setup(LoggingMiddleware()) -download_gecko_driver() -driver = configure_firefox_driver() - stations_cb = CallbackData('station', 'direction') +executor = ThreadPoolExecutor(10) + def get_keyboard() -> types.InlineKeyboardMarkup: """ @@ -44,12 +39,16 @@ async def home_office( query: types.CallbackQuery, callback_data: dict[str, str] ) -> types.Message: - text = parse_site( - driver=driver, - url='https://yandex.ru/maps/213/moscow/stops/stop__9640740/' - '?l=masstransit&ll=37.527754%2C55.823507&tab=overview&z=21', - message='Остановка Б. Академическая ул, д. 15', + driver = get_driver() + + url = ( + 'https://yandex.ru/maps/213/moscow/stops/stop__9640740/' + '?l=masstransit&ll=37.527754%2C55.823507&tab=overview&z=21' ) + message = 'Остановка Б. Академическая ул, д. 15' + + loop = asyncio.get_running_loop() + text = await loop.run_in_executor(executor, parse_site, url, message, driver) return await bot.send_message( query.message.chat.id, text, reply_markup=get_keyboard() @@ -61,12 +60,16 @@ async def office_home( query: types.CallbackQuery, callback_data: dict[str, str] ) -> types.Message: - text = parse_site( - driver=driver, - url='https://yandex.ru/maps/213/moscow/stops/stop__9640288/?' - 'l=masstransit&ll=37.505338%2C55.800160&tab=overview&z=211', - message='Остановка Улица Алабяна', + driver = get_driver() + + url = ( + 'https://yandex.ru/maps/213/moscow/stops/stop__9640288/' + '?l=masstransit&ll=37.505338%2C55.800160&tab=overview&z=211' ) + message = 'Остановка Улица Алабяна' + + loop = asyncio.get_running_loop() + text = await loop.run_in_executor(executor, parse_site, url, message, driver) return await bot.send_message( query.message.chat.id, text, reply_markup=get_keyboard() @@ -86,12 +89,16 @@ async def echo(message: types.Message) -> types.Message: async def morning_bus_mailing(chat_ids: list[int]) -> None: - text = parse_site( - driver=driver, - url='https://yandex.ru/maps/213/moscow/stops/stop__9640740/' - '?l=masstransit&ll=37.527754%2C55.823507&tab=overview&z=21', - message='Остановка Б. Академическая ул, д. 15', + driver = get_driver() + + url = ( + 'https://yandex.ru/maps/213/moscow/stops/stop__9640740/' + '?l=masstransit&ll=37.527754%2C55.823507&tab=overview&z=21' ) + message = 'Остановка Б. Академическая ул, д. 15' + + loop = asyncio.get_running_loop() + text = await loop.run_in_executor(executor, parse_site, url, message, driver) await asyncio.gather( *[ bot.send_message( diff --git a/app/core/parse_web.py b/app/core/parse_web.py index 6ca8a7a..974eaca 100644 --- a/app/core/parse_web.py +++ b/app/core/parse_web.py @@ -7,10 +7,14 @@ import wget from app.core.logger import logger from app.settings import BASE_DIR, GECKO_DRIVER_VERSION from selenium import webdriver -from selenium.common.exceptions import NoSuchElementException, WebDriverException +from selenium.common.exceptions import ( + NoSuchElementException, + StaleElementReferenceException, + WebDriverException, +) from selenium.webdriver.firefox import options from selenium.webdriver.firefox.service import Service -from selenium.webdriver.firefox.webdriver import WebDriver +from selenium.webdriver.firefox.webdriver import RemoteWebDriver, WebDriver def download_gecko_driver() -> None: @@ -52,7 +56,7 @@ def parse_site(url: str, message: str, driver: WebDriver | None = None) -> str: logger.error('Driver is not configured') return 'Что-то пошло не так. :( Драйвер Firefox не сконфигурирован.' driver.get(url) - time.sleep(1) + time.sleep(5) elements = driver.find_elements( by='class name', value='masstransit-vehicle-snippet-view' ) @@ -68,9 +72,6 @@ def parse_site(url: str, message: str, driver: WebDriver | None = None) -> str: bus_300_arrival = element.find_element( by='class name', value='masstransit-prognoses-view__title-text' ) - except NoSuchElementException: - pass - try: bus_t19 = element.find_element( by='css selector', value='[aria-label="т19"]' ) @@ -79,6 +80,9 @@ def parse_site(url: str, message: str, driver: WebDriver | None = None) -> str: ) except NoSuchElementException: pass + except StaleElementReferenceException: + pass + answer = f'{message}\n\n' if not all([bus_300, bus_t19]) or not all([bus_300_arrival, bus_t19_arrival]): return 'Автобусов 300 или Т19 не найдено. \n\nСмотри на карте :)' @@ -86,4 +90,14 @@ def parse_site(url: str, message: str, driver: WebDriver | None = None) -> str: answer += f'Автобус {bus_300.text} - {bus_300_arrival.text}\n' if bus_t19 and bus_t19_arrival: answer += f'Автобус {bus_t19.text} - {bus_t19_arrival.text}' + print(answer) return answer + + +def get_driver() -> RemoteWebDriver: + opt = options.Options() + opt.headless = True + driver = RemoteWebDriver( + command_executor='http://main-server.lan:4444/wd/hub', options=opt + ) + return driver