From 1ecf95631dc6320667d7de7e632ac06a809b8cf4 Mon Sep 17 00:00:00 2001 From: Dmitry Afanasyev <71835315+Balshgit@users.noreply.github.com> Date: Fri, 22 Sep 2023 02:39:17 +0300 Subject: [PATCH] add queue tests --- .github/workflows/check-lint.yml | 2 +- .github/workflows/poetry-test.yml | 5 +- README.md | 42 ++++++------ app/api/system/controllers.py | 3 +- app/core/bot.py | 30 +++------ app/main.py | 20 +++--- app/routers.py | 9 ++- scripts/healthcheck_bot.service | 8 +-- settings/.env.ci.runtests | 21 ++++-- settings/.env.local.runtests | 20 ++++++ settings/.env.staging | 8 --- settings/.env.template | 2 +- settings/config.py | 20 ++++-- tests/integration/bot/conftest.py | 18 +++-- tests/integration/bot/networking.py | 8 +++ tests/integration/bot/test_bot_updates.py | 82 +++++++++++++++++++++++ 16 files changed, 212 insertions(+), 86 deletions(-) delete mode 100644 settings/.env.staging diff --git a/.github/workflows/check-lint.yml b/.github/workflows/check-lint.yml index 28728b3..5a1bbae 100644 --- a/.github/workflows/check-lint.yml +++ b/.github/workflows/check-lint.yml @@ -3,7 +3,7 @@ name: lint on: push: branches-ignore: - - test + - develop tags-ignore: - "*" pull_request: diff --git a/.github/workflows/poetry-test.yml b/.github/workflows/poetry-test.yml index 2924f81..7ec3794 100644 --- a/.github/workflows/poetry-test.yml +++ b/.github/workflows/poetry-test.yml @@ -3,13 +3,16 @@ name: test on: push: branches-ignore: - - test + - develop tags-ignore: - "*" pull_request: branches: - 'release/**' +env: + STAGE: runtests + jobs: test: runs-on: ubuntu-latest diff --git a/README.md b/README.md index 44898d7..cab2c5d 100644 --- a/README.md +++ b/README.md @@ -1,20 +1,20 @@ -# MosGotTrans bot -Бот для получения расписания конкретных автобусов для конкретных остановок +# Chat gpt bot +Бот для запросов в chatgpt -Использует **Selenium** для парсинга сайта "яндекс карты" +Использует **Selenium** и API chatgpt для запросов ## Install & Update install service - sudo cp scripts/mosgortrans.service /etc/systemd/system + sudo cp scripts/chat-gpt.service /etc/systemd/system ```bash -cd ~/PycharmProjects/mosgortrans -sudo systemctl stop healthcheck_bot.service -git pull balshgit master -udo rsync -a --delete --progress ~/mosgortrans/* /opt/mosgortrans/ --exclude .git -sudo systemctl start healthcheck_bot.service +cd ~/PycharmProjects/chat_gpt_bot +sudo systemctl stop chat_gpt_bot.service +git pull balshgit main +sudo rsync -a --delete --progress ~/PycharmProjects/chat_gpt_bot/* /opt/chat_gpt_bot/ --exclude .git +sudo systemctl start chat_gpt_bot.service ``` ## Local start @@ -22,6 +22,10 @@ sudo systemctl start healthcheck_bot.service python main.py ``` +```shell + poetry run uvicorn --host 0.0.0.0 --factory app.main:create_app --port 8000 --reload --reload-dir=app --reload-dir=settings +``` + - set `START_WITH_WEBHOOK` to blank ## Delete or set webhook manually @@ -33,26 +37,24 @@ methods: - set -## Local development clean: - -```bash -killall geckodriver -killall firefox -killall python -``` - ## Tests ```bash -cd tests -SELENOIDTEST=1 docker-compose run test-bot python -m pytest tests/bot/test_bot_selenoid.py::test_selenoid_text -vv +poetry run pytest ``` +## Docs +Docs can be found at + +- {domain}/{url_prefix}/{api_prefix}/docs +- {domain}/{url_prefix}/{api_prefix}/redoc + +on local start can be found at http://localhost/gpt/api/docs + ## Help article [Пишем асинхронного Телеграм-бота](https://habr.com/ru/company/kts/blog/598575/) -[fast_api_aiogram](https://programtalk.com/vs4/python/daya0576/he-weather-bot/telegram_bot/dependencies.py/) ## TODO diff --git a/app/api/system/controllers.py b/app/api/system/controllers.py index 2e705b2..33c5a95 100644 --- a/app/api/system/controllers.py +++ b/app/api/system/controllers.py @@ -19,10 +19,11 @@ async def healthcheck() -> ORJSONResponse: @router.post( - f"/{settings.bot_webhook_url}", + f"/{settings.TELEGRAM_API_TOKEN}", name="system:process_bot_updates", status_code=status.HTTP_202_ACCEPTED, summary="process bot updates", + include_in_schema=False, ) async def process_bot_updates(request: Request) -> ORJSONResponse: await request.app.state.queue.put_updates_on_queue(request) diff --git a/app/core/bot.py b/app/core/bot.py index b12964c..e31d67c 100644 --- a/app/core/bot.py +++ b/app/core/bot.py @@ -1,6 +1,5 @@ import asyncio import os -import time from asyncio import Queue, sleep from dataclasses import dataclass from functools import cached_property @@ -10,18 +9,17 @@ from fastapi import Request, Response from telegram import Update from telegram.ext import Application, CommandHandler, ContextTypes -from app.constants import API_PREFIX -from settings.config import Settings +from settings.config import AppSettings class BotApplication: - def __init__(self, settings: Settings, start_with_webhook: bool = False) -> None: + def __init__(self, settings: AppSettings) -> None: self.application: Application = ( # type: ignore Application.builder().token(token=settings.TELEGRAM_API_TOKEN).build() ) self.add_handlers() self.settings = settings - self.start_with_webhook = start_with_webhook + self.start_with_webhook = settings.START_WITH_WEBHOOK async def set_webhook(self) -> None: await self.application.initialize() @@ -39,11 +37,7 @@ class BotApplication: await update.message.reply_text( "Help!", disable_notification=True, - api_kwargs={ - "text": "Hello World", - "date": int(time.time()) + 30, - "schedule_date": int(time.time()) + 30, - }, + api_kwargs={"text": "Hello World"}, ) return None @@ -60,25 +54,20 @@ class BotApplication: @cached_property def webhook_url(self) -> str: - return os.path.join( - self.settings.WEBHOOK_HOST.strip("/"), - API_PREFIX.strip("/"), - self.settings.URL_PREFIX.strip("/"), - self.settings.TELEGRAM_API_TOKEN.strip("/"), - ) + return os.path.join(self.settings.DOMAIN.strip("/"), self.settings.bot_webhook_url.strip("/")) @dataclass class BotQueue: - bot_app: Application # type: ignore[type-arg] + bot_app: BotApplication queue: Queue = asyncio.Queue() # type: ignore[type-arg] async def put_updates_on_queue(self, request: Request) -> Response: """ - Listen {URL_PREFIX}/{TELEGRAM_WEB_TOKEN} path and proxy post request to bot + Listen /{URL_PREFIX}/{API_PREFIX}/{TELEGRAM_WEB_TOKEN} path and proxy post request to bot """ data = await request.json() - tg_update = Update.de_json(data=data, bot=self.bot_app.bot) + tg_update = Update.de_json(data=data, bot=self.bot_app.application.bot) self.queue.put_nowait(tg_update) return Response(status_code=HTTPStatus.ACCEPTED) @@ -86,6 +75,5 @@ class BotQueue: async def get_updates_from_queue(self) -> None: while True: update = await self.queue.get() - print(update) - await self.bot_app.process_update(update) + await self.bot_app.application.process_update(update) await sleep(0) diff --git a/app/main.py b/app/main.py index 9062f68..b637d0d 100644 --- a/app/main.py +++ b/app/main.py @@ -8,7 +8,7 @@ from loguru import logger from app.core.bot import BotApplication, BotQueue from app.routers import api_router -from settings.config import Settings, get_settings +from settings.config import AppSettings, get_settings logger.remove() logger.add( @@ -20,18 +20,18 @@ logger.add( class Application: - def __init__(self, settings: Settings, bot_app: BotApplication) -> None: + def __init__(self, settings: AppSettings, bot_app: BotApplication) -> None: self.app = FastAPI( - title="Health check bot", - description="Bot which check all services are working", + title="Chat gpt bot", + description="Bot for proxy to chat gpt in telegram", version="0.0.3", - docs_url=f"{settings.URL_PREFIX}/docs", - redoc_url=f"{settings.URL_PREFIX}/redocs", - openapi_url=f"{settings.URL_PREFIX}/api/openapi.json", + docs_url="/" + "/".join([settings.api_prefix.strip("/"), "docs"]), + redoc_url="/" + "/".join([settings.api_prefix.strip("/"), "redocs"]), + openapi_url="/" + "/".join([settings.api_prefix.strip("/"), "openapi.json"]), default_response_class=UJSONResponse, ) self.app.state.settings = settings - self.app.state.queue = BotQueue(bot_app=bot_app.application) + self.app.state.queue = BotQueue(bot_app=bot_app) self.bot_app = bot_app self.app.include_router(api_router) @@ -58,9 +58,9 @@ class Application: await asyncio.gather(self.bot_app.delete_webhook(), self.bot_app.shutdown()) -def create_app(settings: Settings | None = None) -> FastAPI: +def create_app(settings: AppSettings | None = None) -> FastAPI: settings = settings or get_settings() - bot_app = BotApplication(settings=settings, start_with_webhook=settings.START_WITH_WEBHOOK) + bot_app = BotApplication(settings=settings) return Application(settings=settings, bot_app=bot_app).fastapi_app diff --git a/app/routers.py b/app/routers.py index 2a9f0de..9866cc0 100644 --- a/app/routers.py +++ b/app/routers.py @@ -2,9 +2,14 @@ from fastapi import APIRouter from fastapi.responses import ORJSONResponse from app.api.system.controllers import router as system_router -from app.constants import API_PREFIX +from settings.config import get_settings -api_router = APIRouter(prefix=API_PREFIX, default_response_class=ORJSONResponse) +settings = get_settings() + +api_router = APIRouter( + prefix=settings.api_prefix, + default_response_class=ORJSONResponse, +) api_router.include_router(system_router, tags=["system"]) diff --git a/scripts/healthcheck_bot.service b/scripts/healthcheck_bot.service index 15ac8fa..5477e58 100644 --- a/scripts/healthcheck_bot.service +++ b/scripts/healthcheck_bot.service @@ -1,11 +1,11 @@ [Unit] -Description=Healthcheck bot +Description=Chat-gpt bot Wants=network-online.target After=network-online.target [Service] Restart=always -WorkingDirectory=/opt/healthcheck_bot -ExecStart=/usr/local/bin/docker-compose -f /opt/healthcheck_bot/docker-compose.yml up -ExecStop=/usr/local/bin/docker-compose -f /opt/healthcheck_bot/docker-compose.yml down +WorkingDirectory=/opt/chat_gpt_bot +ExecStart=bash -c "docker compose -f /opt/chat_gpt_bot/docker-compose.yml up" +ExecStop=bash -c "docker compose -f /opt/chat_gpt_bot/docker-compose.yml down" [Install] WantedBy=multi-user.target \ No newline at end of file diff --git a/settings/.env.ci.runtests b/settings/.env.ci.runtests index 858ca79..c821656 100644 --- a/settings/.env.ci.runtests +++ b/settings/.env.ci.runtests @@ -3,8 +3,19 @@ STAGE="runtests" APP_HOST="0.0.0.0" APP_PORT="8000" -POSTGRES_HOST="postgres" -POSTGRES_PORT="5432" -POSTGRES_DB="relevancer" -POSTGRES_USER="user" -POSTGRES_PASSWORD="postgrespwd" +USER="web" + +TELEGRAM_API_TOKEN="123456789:AABBCCDDEEFFaabbccddeeff-1234567890" + +# webhook settings +DOMAIN="http://localhost" +URL_PREFIX= + +# set to true to start with webhook. Else bot will start on polling method +START_WITH_WEBHOOK="true" + +# quantity of workers for uvicorn +WORKERS_COUNT=1 +# Enable uvicorn reloading +RELOAD="true" +DEBUG="true" diff --git a/settings/.env.local.runtests b/settings/.env.local.runtests index ff01678..c821656 100644 --- a/settings/.env.local.runtests +++ b/settings/.env.local.runtests @@ -1 +1,21 @@ STAGE="runtests" + +APP_HOST="0.0.0.0" +APP_PORT="8000" + +USER="web" + +TELEGRAM_API_TOKEN="123456789:AABBCCDDEEFFaabbccddeeff-1234567890" + +# webhook settings +DOMAIN="http://localhost" +URL_PREFIX= + +# set to true to start with webhook. Else bot will start on polling method +START_WITH_WEBHOOK="true" + +# quantity of workers for uvicorn +WORKERS_COUNT=1 +# Enable uvicorn reloading +RELOAD="true" +DEBUG="true" diff --git a/settings/.env.staging b/settings/.env.staging deleted file mode 100644 index 7d19618..0000000 --- a/settings/.env.staging +++ /dev/null @@ -1,8 +0,0 @@ -APP_HOST="0.0.0.0" -APP_PORT="8000" - -POSTGRES_HOST="postgres" -POSTGRES_PORT="5432" -POSTGRES_DB="relevancer" -POSTGRES_USER="user" -POSTGRES_PASSWORD="postgrespwd" diff --git a/settings/.env.template b/settings/.env.template index f94d7aa..aad5cd4 100644 --- a/settings/.env.template +++ b/settings/.env.template @@ -8,7 +8,7 @@ USER="web" TELEGRAM_API_TOKEN="123456789:AABBCCDDEEFFaabbccddeeff-1234567890" # webhook settings -WEBHOOK_HOST="https://mydomain.com" +DOMAIN="https://mydomain.com" URL_PREFIX="/gpt" # set to true to start with webhook. Else bot will start on polling method diff --git a/settings/config.py b/settings/config.py index 2795d3f..97ade5a 100644 --- a/settings/config.py +++ b/settings/config.py @@ -5,6 +5,8 @@ from pathlib import Path from dotenv import load_dotenv from pydantic_settings import BaseSettings +from app.constants import API_PREFIX + BASE_DIR = Path(__file__).parent.parent SHARED_DIR = BASE_DIR.resolve().joinpath("shared") SHARED_DIR.mkdir(exist_ok=True) @@ -23,10 +25,10 @@ if environ.get("STAGE") == "runtests": load_dotenv(env_path, override=True) -class Settings(BaseSettings): +class AppSettings(BaseSettings): """Application settings.""" - PROJECT_NAME: str = "healthcheck bot" + PROJECT_NAME: str = "chat gpt bot" APP_HOST: str = "0.0.0.0" APP_PORT: int = 8000 STAGE: str = "dev" @@ -35,7 +37,7 @@ class Settings(BaseSettings): TELEGRAM_API_TOKEN: str = "123456789:AABBCCDDEEFFaabbccddeeff-1234567890" # webhook settings START_WITH_WEBHOOK: bool = False - WEBHOOK_HOST: str = "https://mydomain.com" + DOMAIN: str = "https://localhost" URL_PREFIX: str = "" # quantity of workers for uvicorn @@ -43,13 +45,19 @@ class Settings(BaseSettings): # Enable uvicorn reloading RELOAD: bool = False + @cached_property + def api_prefix(self) -> str: + if self.URL_PREFIX: + return "/" + "/".join([self.URL_PREFIX.strip("/"), API_PREFIX.strip("/")]) + return API_PREFIX + @cached_property def bot_webhook_url(self) -> str: - return "/" + self.TELEGRAM_API_TOKEN + return "/".join([self.api_prefix, self.TELEGRAM_API_TOKEN]) class Config: case_sensitive = True -def get_settings() -> Settings: - return Settings() +def get_settings() -> AppSettings: + return AppSettings() diff --git a/tests/integration/bot/conftest.py b/tests/integration/bot/conftest.py index 9753bde..a1e90bf 100644 --- a/tests/integration/bot/conftest.py +++ b/tests/integration/bot/conftest.py @@ -17,11 +17,16 @@ from telegram.ext import Application, ApplicationBuilder, Defaults, ExtBot from app.core.bot import BotApplication from app.main import Application as AppApplication -from settings.config import get_settings +from settings.config import AppSettings, get_settings from tests.integration.bot.networking import NonchalantHttpxRequest from tests.integration.factories.bot import BotInfoFactory +@pytest.fixture(scope="session") +def test_settings() -> AppSettings: + return get_settings() + + class PytestExtBot(ExtBot): # type: ignore def __init__(self, *args: Any, **kwargs: Any) -> None: super().__init__(*args, **kwargs) @@ -213,12 +218,13 @@ async def bot_application(bot_info: dict[str, Any]) -> AsyncGenerator[Any, None] @pytest_asyncio.fixture(scope="session") -async def main_application(bot_application: PytestApplication) -> FastAPI: - settings = get_settings() - bot_app = BotApplication(settings=settings) +async def main_application( + bot_application: PytestApplication, test_settings: AppSettings +) -> AsyncGenerator[FastAPI, None]: + bot_app = BotApplication(settings=test_settings) bot_app.application = bot_application - fast_api_app = AppApplication(settings=settings, bot_app=bot_app).fastapi_app - return fast_api_app + fast_api_app = AppApplication(settings=test_settings, bot_app=bot_app).fastapi_app + yield fast_api_app @pytest_asyncio.fixture() diff --git a/tests/integration/bot/networking.py b/tests/integration/bot/networking.py index af6b686..beda6f7 100644 --- a/tests/integration/bot/networking.py +++ b/tests/integration/bot/networking.py @@ -99,3 +99,11 @@ async def send_webhook_message( data=payload, # type: ignore headers=headers, ) + + +class MockedRequest: + def __init__(self, data: dict[str, Any]) -> None: + self.data = data + + async def json(self) -> dict[str, Any]: + return self.data diff --git a/tests/integration/bot/test_bot_updates.py b/tests/integration/bot/test_bot_updates.py index 03d58ed..6b1160f 100644 --- a/tests/integration/bot/test_bot_updates.py +++ b/tests/integration/bot/test_bot_updates.py @@ -1,6 +1,15 @@ +import asyncio +import time +from asyncio import AbstractEventLoop +from typing import Any + import pytest from httpx import AsyncClient +from app.core.bot import BotApplication, BotQueue +from app.main import Application +from tests.integration.bot.networking import MockedRequest + pytestmark = [ pytest.mark.asyncio, ] @@ -10,3 +19,76 @@ async def test_bot_updates(rest_client: AsyncClient) -> None: response = await rest_client.get("/api/healthcheck") assert response.status_code == 200 + + +async def test_bot_webhook_endpoint( + rest_client: AsyncClient, +) -> None: + response = await rest_client.post( + url="/api/123456789:AABBCCDDEEFFaabbccddeeff-1234567890", + json={ + "update_id": 957250703, + "message": { + "message_id": 417070387, + "from": { + "id": 1000, + "is_bot": "false", + "first_name": "William", + "last_name": "Dalton", + "username": "bolshakovfortunat", + "language_code": "ru", + }, + "chat": { + "id": 1, + "first_name": "Gabrielle", + "last_name": "Smith", + "username": "arefi_2019", + "type": "private", + }, + "date": time.time(), + "text": "/chatid", + "entities": [{"type": "bot_command", "offset": 0, "length": 7}], + }, + }, + ) + assert response.status_code == 202 + + +async def test_bot_queue( + bot: BotApplication, + bot_application: Any, + main_application: Application, + event_loop: AbstractEventLoop, +) -> None: + bot.application = bot_application + bot_queue = BotQueue(bot_app=bot) + event_loop.create_task(bot_queue.get_updates_from_queue()) + mocked_request = MockedRequest( + { + "update_id": 957250703, + "message": { + "message_id": 417070387, + "from": { + "id": 1000, + "is_bot": "false", + "first_name": "William", + "last_name": "Dalton", + "username": "bolshakovfortunat", + "language_code": "ru", + }, + "chat": { + "id": 1, + "first_name": "Gabrielle", + "last_name": "Smith", + "username": "arefi_2019", + "type": "private", + }, + "date": time.time(), + "text": "/chatid", + "entities": [{"type": "bot_command", "offset": 0, "length": 7}], + }, + } + ) + await bot_queue.put_updates_on_queue(mocked_request) # type: ignore + await asyncio.sleep(1) + assert bot_queue.queue.empty()