From 22de340fddf8d8321423cdde08683616f78b69bd Mon Sep 17 00:00:00 2001 From: Dmitry Afanasyev Date: Sun, 14 Aug 2022 02:24:26 +0300 Subject: [PATCH] run in executor --- .github/workflows/poetry-test.yml | 2 +- app/core/bot.py | 24 ++++++++++---- app/core/parse_web.py | 2 +- app/tests/factories.py | 18 ++++++++++ app/tests/{dataset.py => models.py} | 14 ++++++++ app/tests/test_parse_web.py | 7 ++-- poetry.lock | 51 ++++++++++++++++++++++++++++- pyproject.toml | 3 ++ 8 files changed, 109 insertions(+), 12 deletions(-) create mode 100644 app/tests/factories.py rename app/tests/{dataset.py => models.py} (97%) diff --git a/.github/workflows/poetry-test.yml b/.github/workflows/poetry-test.yml index 2bf23e4..e50c200 100644 --- a/.github/workflows/poetry-test.yml +++ b/.github/workflows/poetry-test.yml @@ -11,7 +11,7 @@ on: - 'release/**' jobs: - build: + test: runs-on: ubuntu-latest steps: #---------------------------------------------- diff --git a/app/core/bot.py b/app/core/bot.py index a4282a6..0475e9b 100644 --- a/app/core/bot.py +++ b/app/core/bot.py @@ -1,10 +1,12 @@ import asyncio +from concurrent.futures.thread import ThreadPoolExecutor from aiogram import Bot, types from aiogram.contrib.middlewares.logging import LoggingMiddleware from aiogram.dispatcher import Dispatcher from aiogram.dispatcher.webhook import SendMessage from aiogram.utils.callback_data import CallbackData + from core.parse_web import configure_firefox_driver, download_gecko_driver, parse_site from settings import API_TOKEN @@ -15,6 +17,8 @@ dispatcher.middleware.setup(LoggingMiddleware()) download_gecko_driver() driver = configure_firefox_driver() +executor = ThreadPoolExecutor(5) + stations_cb = CallbackData('station', 'direction') @@ -40,12 +44,20 @@ async def home_office( query: types.CallbackQuery, callback_data: dict[str, str] ) -> SendMessage: - text = parse_site( - driver=driver, - url='https://yandex.ru/maps/213/moscow/stops/stop__9640740/' - '?l=masstransit&ll=37.527754%2C55.823507&tab=overview&z=21', - message='Остановка Б. Академическая ул, д. 15', - ) + url = ('https://yandex.ru/maps/213/moscow/stops/stop__9640740/' + '?l=masstransit&ll=37.527754%2C55.823507&tab=overview&z=21' + ) + message = 'Остановка Б. Академическая ул, д. 15' + + loop = asyncio.get_running_loop() + text = await loop.run_in_executor(executor, parse_site, driver, url, message) + + # text = parse_site( + # driver=driver, + # url='https://yandex.ru/maps/213/moscow/stops/stop__9640740/' + # '?l=masstransit&ll=37.527754%2C55.823507&tab=overview&z=21', + # message='Остановка Б. Академическая ул, д. 15', + # ) return SendMessage(query.message.chat.id, text, reply_markup=get_keyboard()) diff --git a/app/core/parse_web.py b/app/core/parse_web.py index 2c2c87a..07dc8dc 100644 --- a/app/core/parse_web.py +++ b/app/core/parse_web.py @@ -44,7 +44,7 @@ def configure_firefox_driver(private_window: bool = False) -> WebDriver: return firefox_driver -async def parse_site(driver: WebDriver, url: str, message: str) -> str: +def parse_site(driver: WebDriver, url: str, message: str) -> str: driver.get(url) time.sleep(4) elements = driver.find_elements( diff --git a/app/tests/factories.py b/app/tests/factories.py new file mode 100644 index 0000000..9038bf2 --- /dev/null +++ b/app/tests/factories.py @@ -0,0 +1,18 @@ +import factory +from faker import Faker + +from tests.models import User + +faker = Faker('ru_RU') + + +class UserFactory(factory.Factory): + id = factory.Sequence(lambda n: 1000 + n) + is_bot = False + first_name = factory.Faker('first_name') + last_name = factory.Faker('last_name') + username = faker.profile(fields=['username'])['username'] + language_code = 'ru' + + class Meta: + model = User diff --git a/app/tests/dataset.py b/app/tests/models.py similarity index 97% rename from app/tests/dataset.py rename to app/tests/models.py index 04394c3..27d6f4d 100644 --- a/app/tests/dataset.py +++ b/app/tests/models.py @@ -1,6 +1,20 @@ """" Dict data set for Telegram message types """ +from typing import NamedTuple, Any + + +class User(NamedTuple): + id: int + is_bot: bool + first_name: str | None + last_name: str | None + username: str | None + language_code: str + + def as_dict(self) -> dict[str, Any]: + return self._asdict() + USER = { "id": 12345678, diff --git a/app/tests/test_parse_web.py b/app/tests/test_parse_web.py index 0d92fd4..8b2c192 100644 --- a/app/tests/test_parse_web.py +++ b/app/tests/test_parse_web.py @@ -1,7 +1,7 @@ import pytest from aiogram import Bot, types from app.tests.conftest import FakeTelegram -from app.tests.dataset import USER +from tests.factories import UserFactory pytestmark = [ pytest.mark.asyncio, @@ -9,9 +9,10 @@ pytestmark = [ async def test_parse_site(bot: Bot) -> None: - user = types.User(**USER) + tg_user = UserFactory().as_dict() + user = types.User(**tg_user) - async with FakeTelegram(message_data=USER): + async with FakeTelegram(message_data=tg_user): result = await bot.me assert result == user diff --git a/poetry.lock b/poetry.lock index efe37ae..73f1a1c 100644 --- a/poetry.lock +++ b/poetry.lock @@ -393,6 +393,32 @@ category = "dev" optional = false python-versions = "*" +[[package]] +name = "factory-boy" +version = "3.2.1" +description = "A versatile test fixtures replacement based on thoughtbot's factory_bot for Ruby." +category = "main" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +Faker = ">=0.7.0" + +[package.extras] +dev = ["coverage", "django", "flake8", "isort", "pillow", "sqlalchemy", "mongoengine", "wheel (>=0.32.0)", "tox", "zest.releaser"] +doc = ["sphinx", "sphinx-rtd-theme", "sphinxcontrib-spelling"] + +[[package]] +name = "faker" +version = "14.0.0" +description = "Faker is a Python package that generates fake data for you." +category = "main" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +python-dateutil = ">=2.4" + [[package]] name = "filelock" version = "3.8.0" @@ -1202,6 +1228,17 @@ python-versions = "*" [package.dependencies] pytest = ">=3.6.0" +[[package]] +name = "python-dateutil" +version = "2.8.2" +description = "Extensions to the standard Python datetime module" +category = "main" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" + +[package.dependencies] +six = ">=1.5" + [[package]] name = "python-decouple" version = "3.6" @@ -1657,7 +1694,7 @@ multidict = ">=4.0" [metadata] lock-version = "1.1" python-versions = "^3.10" -content-hash = "429b7926e38d696263b3886755fbb33c3ac01faae3d2f124f7d5d38f6b20361e" +content-hash = "f6a0f7174a72427f856464538c02eb4b909b78d3ed94afda1f44ae996749c210" [metadata.files] aiogram = [ @@ -1997,6 +2034,14 @@ executing = [ {file = "executing-0.10.0-py2.py3-none-any.whl", hash = "sha256:9c745f80cda11eb22b62cbecf21156491a794eb56ab06f9d286a44e62822b24e"}, {file = "executing-0.10.0.tar.gz", hash = "sha256:d1cd87c2e371e9966261410c5b3769d6df2f9e4a79a83eebd2662dd3388f9833"}, ] +factory-boy = [ + {file = "factory_boy-3.2.1-py2.py3-none-any.whl", hash = "sha256:eb02a7dd1b577ef606b75a253b9818e6f9eaf996d94449c9d5ebb124f90dc795"}, + {file = "factory_boy-3.2.1.tar.gz", hash = "sha256:a98d277b0c047c75eb6e4ab8508a7f81fb03d2cb21986f627913546ef7a2a55e"}, +] +faker = [ + {file = "Faker-14.0.0-py3-none-any.whl", hash = "sha256:f1558ecb1770d8c871ea01cc2edc7b5e86148b0fa0466731f0e1e8953165d179"}, + {file = "Faker-14.0.0.tar.gz", hash = "sha256:0c7d283a96c49af64fe319f70d2b68927873c9173e922f8eda6001e7757cb63b"}, +] filelock = [ {file = "filelock-3.8.0-py3-none-any.whl", hash = "sha256:617eb4e5eedc82fc5f47b6d61e4d11cb837c56cb4544e39081099fa17ad109d4"}, {file = "filelock-3.8.0.tar.gz", hash = "sha256:55447caa666f2198c5b6b13a26d2084d26fa5b115c00d065664b2124680c4edc"}, @@ -2456,6 +2501,10 @@ pytest-timeout = [ {file = "pytest-timeout-1.4.2.tar.gz", hash = "sha256:20b3113cf6e4e80ce2d403b6fb56e9e1b871b510259206d40ff8d609f48bda76"}, {file = "pytest_timeout-1.4.2-py2.py3-none-any.whl", hash = "sha256:541d7aa19b9a6b4e475c759fd6073ef43d7cdc9a92d95644c260076eb257a063"}, ] +python-dateutil = [ + {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, + {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, +] python-decouple = [ {file = "python-decouple-3.6.tar.gz", hash = "sha256:2838cdf77a5cf127d7e8b339ce14c25bceb3af3e674e039d4901ba16359968c7"}, {file = "python_decouple-3.6-py3-none-any.whl", hash = "sha256:6cf502dc963a5c642ea5ead069847df3d916a6420cad5599185de6bab11d8c2e"}, diff --git a/pyproject.toml b/pyproject.toml index 7ad9aea..071f242 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,6 +15,9 @@ python-decouple = "^3.6" apscheduler = "^3.9.1" SQLAlchemy = {version = "^1.4", extras = ["mypy", "asyncio"]} +factory-boy = "^3.2.1" +Faker = "^14.0.0" + [tool.poetry.dev-dependencies] ipython = "^8.2.0" pre-commit = "^2.14.0"