mirror of
https://github.com/Balshgit/mosgortrans.git
synced 2025-09-11 13:00:40 +03:00
reworked web_parser to class
This commit is contained in:
parent
591d5ea01a
commit
0c326fd12d
@ -5,7 +5,7 @@ from aiogram import Bot, types
|
||||
from aiogram.contrib.middlewares.logging import LoggingMiddleware
|
||||
from aiogram.dispatcher import Dispatcher
|
||||
from aiogram.utils.callback_data import CallbackData
|
||||
from app.core.parse_web import get_driver, parse_yandex_maps
|
||||
from app.core.parse_web import WebParser
|
||||
from app.settings import TELEGRAM_API_TOKEN
|
||||
|
||||
|
||||
@ -14,7 +14,6 @@ class TransportBot:
|
||||
bot: Bot = Bot(TELEGRAM_API_TOKEN)
|
||||
dispatcher: Dispatcher = Dispatcher(bot)
|
||||
dispatcher.middleware.setup(LoggingMiddleware())
|
||||
|
||||
stations_cb: CallbackData = CallbackData('station', 'direction')
|
||||
|
||||
@staticmethod
|
||||
@ -46,8 +45,8 @@ class TransportBot:
|
||||
async def home_office(
|
||||
query: types.CallbackQuery, callback_data: dict[str, str]
|
||||
) -> types.Message:
|
||||
driver = get_driver()
|
||||
text = parse_yandex_maps(
|
||||
driver = WebParser.get_driver()
|
||||
text = WebParser.parse_yandex_maps(
|
||||
driver=driver,
|
||||
url='https://yandex.ru/maps/213/moscow/stops/stop__9640740/?ll=37.527924%2C55.823470&tab=overview&z=21',
|
||||
message='Остановка Б. Академическая ул, д. 15',
|
||||
@ -62,8 +61,8 @@ class TransportBot:
|
||||
async def office_home(
|
||||
query: types.CallbackQuery, callback_data: dict[str, str]
|
||||
) -> types.Message:
|
||||
driver = get_driver()
|
||||
text = parse_yandex_maps(
|
||||
driver = WebParser.get_driver()
|
||||
text = WebParser.parse_yandex_maps(
|
||||
driver=driver,
|
||||
url='https://yandex.ru/maps/213/moscow/stops/stop__9640288/?ll=37.505402%2C55.800214&tab=overview&z=21',
|
||||
message='Остановка Улица Алабяна',
|
||||
@ -87,8 +86,8 @@ class TransportBot:
|
||||
if not chat_ids:
|
||||
return None
|
||||
|
||||
driver = get_driver()
|
||||
text = parse_yandex_maps(
|
||||
driver = WebParser.get_driver()
|
||||
text = WebParser.parse_yandex_maps(
|
||||
driver=driver,
|
||||
url='https://yandex.ru/maps/213/moscow/stops/stop__9640740/?ll=37.527924%2C55.823470&tab=overview&z=21',
|
||||
message='Остановка Б. Академическая ул, д. 15',
|
||||
|
@ -18,93 +18,97 @@ from selenium.webdriver.firefox.service import Service
|
||||
from selenium.webdriver.firefox.webdriver import RemoteWebDriver, WebDriver
|
||||
|
||||
|
||||
def download_gecko_driver() -> None:
|
||||
gecko_driver = (
|
||||
f'https://github.com/mozilla/geckodriver/releases/download/v{GECKO_DRIVER_VERSION}/'
|
||||
f'geckodriver-v{GECKO_DRIVER_VERSION}-linux64.tar.gz'
|
||||
)
|
||||
|
||||
if not Path(BASE_DIR / 'geckodriver').exists():
|
||||
logger.info(f'Downloading gecodriver v {GECKO_DRIVER_VERSION}...')
|
||||
geckodriver_file = wget.download(
|
||||
url=gecko_driver, out=BASE_DIR.resolve().as_posix()
|
||||
class WebParser:
|
||||
@staticmethod
|
||||
def download_gecko_driver() -> None:
|
||||
gecko_driver_url = (
|
||||
f'https://github.com/mozilla/geckodriver/releases/download/v{GECKO_DRIVER_VERSION}/'
|
||||
f'geckodriver-v{GECKO_DRIVER_VERSION}-linux64.tar.gz'
|
||||
)
|
||||
|
||||
with tarfile.open(geckodriver_file) as tar:
|
||||
tar.extractall(BASE_DIR)
|
||||
os.remove(f'{BASE_DIR / "geckodriver"}-v{GECKO_DRIVER_VERSION}-linux64.tar.gz')
|
||||
logger.info(f'\ngeckodriver has been downloaded to folder {BASE_DIR}')
|
||||
|
||||
|
||||
def configure_firefox_driver(private_window: bool = False) -> WebDriver | None:
|
||||
opt = options.Options()
|
||||
opt.headless = True
|
||||
opt.add_argument('-profile')
|
||||
opt.add_argument(f'{Path.home()}/snap/firefox/common/.mozilla/firefox')
|
||||
if private_window:
|
||||
opt.set_preference("browser.privatebrowsing.autostart", True)
|
||||
service = Service(executable_path=(BASE_DIR / 'geckodriver').as_posix())
|
||||
try:
|
||||
firefox_driver = webdriver.Firefox(service=service, options=opt)
|
||||
return firefox_driver
|
||||
except WebDriverException:
|
||||
logger.error('Error configuring webdriver. Possible it already configured')
|
||||
return None
|
||||
|
||||
|
||||
def parse_yandex_maps(
|
||||
url: str, message: str, driver: RemoteWebDriver | None = None
|
||||
) -> str:
|
||||
if not driver:
|
||||
logger.error('Driver is not configured')
|
||||
return 'Что-то пошло не так. :( Драйвер Firefox не сконфигурирован.'
|
||||
|
||||
driver.get(url)
|
||||
time.sleep(1)
|
||||
|
||||
bus_arrival: dict[str, str | None] = defaultdict(str)
|
||||
|
||||
try:
|
||||
web_elements = driver.find_elements(
|
||||
by='class name', value='masstransit-vehicle-snippet-view'
|
||||
)
|
||||
for web_element in web_elements:
|
||||
bus = web_element.find_element(
|
||||
by='class name', value='masstransit-vehicle-snippet-view__main-text'
|
||||
if not Path(BASE_DIR / 'geckodriver').exists():
|
||||
logger.info(f'Downloading gecodriver v {GECKO_DRIVER_VERSION}...')
|
||||
geckodriver_file = wget.download(
|
||||
url=gecko_driver_url, out=BASE_DIR.resolve().as_posix()
|
||||
)
|
||||
if bus:
|
||||
bus_arrival_time = web_element.find_element(
|
||||
by='class name',
|
||||
value='masstransit-prognoses-view__title-text',
|
||||
|
||||
with tarfile.open(geckodriver_file) as tar:
|
||||
tar.extractall(BASE_DIR)
|
||||
os.remove(
|
||||
f'{BASE_DIR / "geckodriver"}-v{GECKO_DRIVER_VERSION}-linux64.tar.gz'
|
||||
)
|
||||
logger.info(f'\ngeckodriver has been downloaded to folder {BASE_DIR}')
|
||||
|
||||
@staticmethod
|
||||
def configure_firefox_driver(private_window: bool = False) -> WebDriver | None:
|
||||
opt = options.Options()
|
||||
opt.headless = True
|
||||
opt.add_argument('-profile')
|
||||
opt.add_argument(f'{Path.home()}/snap/firefox/common/.mozilla/firefox')
|
||||
if private_window:
|
||||
opt.set_preference("browser.privatebrowsing.autostart", True)
|
||||
service = Service(executable_path=(BASE_DIR / 'geckodriver').as_posix())
|
||||
try:
|
||||
firefox_driver = webdriver.Firefox(service=service, options=opt)
|
||||
return firefox_driver
|
||||
except WebDriverException:
|
||||
logger.error('Error configuring webdriver. Possible it already configured')
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def parse_yandex_maps(
|
||||
url: str, message: str, driver: RemoteWebDriver | None = None
|
||||
) -> str:
|
||||
if not driver:
|
||||
logger.error('Driver is not configured')
|
||||
return 'Что-то пошло не так. :( Драйвер Firefox не сконфигурирован.'
|
||||
|
||||
driver.get(url)
|
||||
time.sleep(1)
|
||||
|
||||
bus_arrival: dict[str, str | None] = defaultdict(str)
|
||||
|
||||
try:
|
||||
web_elements = driver.find_elements(
|
||||
by='class name', value='masstransit-vehicle-snippet-view'
|
||||
)
|
||||
for web_element in web_elements:
|
||||
bus = web_element.find_element(
|
||||
by='class name', value='masstransit-vehicle-snippet-view__main-text'
|
||||
)
|
||||
match bus.text:
|
||||
case "300":
|
||||
bus_arrival["Автобус 300"] = (
|
||||
bus_arrival_time.text if bus_arrival_time else None
|
||||
)
|
||||
case "т19":
|
||||
bus_arrival["Автобус Т19"] = (
|
||||
bus_arrival_time.text if bus_arrival_time else None
|
||||
)
|
||||
except NoSuchElementException:
|
||||
pass
|
||||
except StaleElementReferenceException:
|
||||
pass
|
||||
if bus:
|
||||
bus_arrival_time = web_element.find_element(
|
||||
by='class name',
|
||||
value='masstransit-prognoses-view__title-text',
|
||||
)
|
||||
match bus.text:
|
||||
case "300":
|
||||
bus_arrival["Автобус 300"] = (
|
||||
bus_arrival_time.text if bus_arrival_time else None
|
||||
)
|
||||
case "т19":
|
||||
bus_arrival["Автобус Т19"] = (
|
||||
bus_arrival_time.text if bus_arrival_time else None
|
||||
)
|
||||
except NoSuchElementException:
|
||||
pass
|
||||
except StaleElementReferenceException:
|
||||
pass
|
||||
|
||||
if not any(bus_arrival.values()):
|
||||
return 'Автобусов 300 или Т19 не найдено. \n\nСмотри на карте :)'
|
||||
if not any(bus_arrival.values()):
|
||||
return 'Автобусов 300 или Т19 не найдено. \n\nСмотри на карте :)'
|
||||
|
||||
answer = f'{message}\n\n'
|
||||
for bus_name, arrival_time in bus_arrival.items():
|
||||
answer += f'{bus_name} - {arrival_time}\n'
|
||||
return answer
|
||||
answer = f'{message}\n\n'
|
||||
for bus_name, arrival_time in bus_arrival.items():
|
||||
answer += f'{bus_name} - {arrival_time}\n'
|
||||
return answer
|
||||
|
||||
|
||||
@timed_cache(seconds=DRIVER_SESSION_TTL)
|
||||
def get_driver() -> RemoteWebDriver:
|
||||
opt = options.Options()
|
||||
opt.headless = True
|
||||
driver = RemoteWebDriver(
|
||||
command_executor='http://selenoid_host:4444/wd/hub', options=opt
|
||||
)
|
||||
return driver
|
||||
@staticmethod
|
||||
@timed_cache(seconds=DRIVER_SESSION_TTL)
|
||||
def get_driver() -> RemoteWebDriver:
|
||||
opt = options.Options()
|
||||
opt.headless = True
|
||||
driver = RemoteWebDriver(
|
||||
command_executor='http://selenoid_host:4444/wd/hub', options=opt
|
||||
)
|
||||
return driver
|
||||
|
@ -4,7 +4,7 @@ import pytest
|
||||
from aiogram import Bot, Dispatcher, types
|
||||
from aiogram.dispatcher.filters.builtin import Command
|
||||
from aiogram.types import Update
|
||||
from app.core.bot import dispatcher
|
||||
from app.core.bot import TransportBot
|
||||
from tests.conftest import FakeTelegram
|
||||
from tests.data.factories import UserFactory
|
||||
|
||||
@ -25,8 +25,8 @@ async def test_parse_yandex_maps(bot: Bot) -> None:
|
||||
|
||||
async def test_command1(bot: Bot) -> None:
|
||||
|
||||
dispatcher.bot = bot
|
||||
handlers = dispatcher.message_handlers.handlers
|
||||
TransportBot.dispatcher.bot = bot
|
||||
handlers = TransportBot.dispatcher.message_handlers.handlers
|
||||
for handler in handlers:
|
||||
handl = list(
|
||||
filter(lambda obj: isinstance(obj.filter, Command), handler.filters)
|
||||
@ -66,5 +66,4 @@ async def test_update(dispatcher_fixture: Dispatcher, bot: Bot) -> None:
|
||||
update = Update(**data)
|
||||
dispatcher_fixture.message_handler()
|
||||
await dispatcher_fixture.process_update(update)
|
||||
|
||||
assert True
|
||||
|
Loading…
x
Reference in New Issue
Block a user