From 28895f3510557e9e591def3c14ab108a6cd11374 Mon Sep 17 00:00:00 2001 From: Dmitry Afanasyev <71835315+Balshgit@users.noreply.github.com> Date: Mon, 8 Jan 2024 21:33:35 +0300 Subject: [PATCH] remove models update from migrations (#80) * add bot models update script * add last question at field * update README.md --- README.md | 4 +- bot_microservice/core/auth/models/users.py | 15 +- bot_microservice/core/auth/repository.py | 5 +- .../core/bot/managment/__init__.py | 0 .../core/bot/managment/update_gpt_models.py | 13 ++ bot_microservice/core/bot/repository.py | 12 ++ bot_microservice/core/bot/services.py | 139 +++++++++--------- bot_microservice/infra/admin.py | 10 +- .../versions/0004_add_chatgpt_models.py | 43 ------ .../versions/0004_add_last_question_at.py | 29 ++++ .../tests/integration/bot/test_bot_updates.py | 7 + .../integration/bot/test_chatgpt_models.py | 30 ++++ .../tests/integration/factories/user.py | 1 + poetry.lock | 21 +-- pyproject.toml | 3 +- scripts/start-bot.sh | 4 +- 16 files changed, 191 insertions(+), 145 deletions(-) create mode 100644 bot_microservice/core/bot/managment/__init__.py create mode 100644 bot_microservice/core/bot/managment/update_gpt_models.py delete mode 100644 bot_microservice/infra/database/migrations/versions/0004_add_chatgpt_models.py create mode 100644 bot_microservice/infra/database/migrations/versions/0004_add_last_question_at.py create mode 100644 bot_microservice/tests/integration/bot/test_chatgpt_models.py diff --git a/README.md b/README.md index 96354b0..e2dbd75 100644 --- a/README.md +++ b/README.md @@ -110,6 +110,7 @@ Init alembic cd bot_microservice alembic revision --autogenerate -m 'create_chatgpt_table' alembic upgrade head +python3 core/bot/managment/update_gpt_models.py ``` @@ -130,6 +131,7 @@ Run migrations ```bash cd ./bot_microservice # alembic root alembic --config ./alembic.ini upgrade head +python3 core/bot/managment/update_gpt_models.py ``` **downgrade to 0001_create_chatgpt_table revision:** @@ -166,4 +168,4 @@ alembic downgrade base - [x] add sentry - [x] add graylog integration and availability to log to file - [x] add user model -- [ ] add messages statistic +- [x] add messages statistic diff --git a/bot_microservice/core/auth/models/users.py b/bot_microservice/core/auth/models/users.py index 7e27f55..a09e531 100644 --- a/bot_microservice/core/auth/models/users.py +++ b/bot_microservice/core/auth/models/users.py @@ -19,9 +19,7 @@ class User(Base): hashed_password: Mapped[str] = mapped_column(String(length=1024), nullable=False) is_active: Mapped[bool] = mapped_column(Boolean, default=True, nullable=False) is_superuser: Mapped[bool] = mapped_column(Boolean, default=False, nullable=False) - created_at: Mapped[datetime] = mapped_column( - TIMESTAMP(timezone=True), index=True, nullable=False, default=datetime.now - ) + created_at: Mapped[datetime] = mapped_column(TIMESTAMP(timezone=True), nullable=False, default=datetime.now) user_question_count: Mapped["UserQuestionCount"] = relationship( "UserQuestionCount", @@ -38,6 +36,12 @@ class User(Base): return self.user_question_count.question_count return 0 + @property + def last_question_at(self) -> str | None: + if self.user_question_count: + return self.user_question_count.last_question_at.strftime("%Y-%m-%d %H:%M:%S") + return None + @classmethod def build( cls, @@ -70,9 +74,7 @@ class AccessToken(Base): user_id = mapped_column(INTEGER, ForeignKey("users.id", ondelete="cascade"), nullable=False) token: Mapped[str] = mapped_column(String(length=42), primary_key=True, default=lambda: str(uuid.uuid4())) - created_at: Mapped[datetime] = mapped_column( - TIMESTAMP(timezone=True), index=True, nullable=False, default=datetime.now - ) + created_at: Mapped[datetime] = mapped_column(TIMESTAMP(timezone=True), nullable=False, default=datetime.now) user: Mapped["User"] = relationship( "User", @@ -94,3 +96,4 @@ class UserQuestionCount(Base): user_id: Mapped[int] = mapped_column(INTEGER, ForeignKey("users.id", ondelete="cascade"), primary_key=True) question_count: Mapped[int] = mapped_column(INTEGER, default=0, nullable=False) + last_question_at: Mapped[datetime] = mapped_column(TIMESTAMP(timezone=True), nullable=False, default=datetime.now) diff --git a/bot_microservice/core/auth/repository.py b/bot_microservice/core/auth/repository.py index 967d5a0..5586316 100644 --- a/bot_microservice/core/auth/repository.py +++ b/bot_microservice/core/auth/repository.py @@ -1,6 +1,6 @@ from dataclasses import dataclass -from sqlalchemy import select +from sqlalchemy import func, select from sqlalchemy.dialects.sqlite import insert from sqlalchemy.orm import load_only @@ -69,7 +69,8 @@ class UserRepository: UserQuestionCount.get_real_column_name( UserQuestionCount.question_count.key ): UserQuestionCount.question_count - + 1 + + 1, + UserQuestionCount.get_real_column_name(UserQuestionCount.last_question_at.key): func.now(), }, ) ) diff --git a/bot_microservice/core/bot/managment/__init__.py b/bot_microservice/core/bot/managment/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/bot_microservice/core/bot/managment/update_gpt_models.py b/bot_microservice/core/bot/managment/update_gpt_models.py new file mode 100644 index 0000000..074acc6 --- /dev/null +++ b/bot_microservice/core/bot/managment/update_gpt_models.py @@ -0,0 +1,13 @@ +import asyncio +import sys +from pathlib import Path + +from loguru import logger + +if __name__ == "__main__": + sys.path.append(str(Path(__file__).parent.parent.parent.parent)) + from core.bot.services import ChatGptService + + chatgpt_service = ChatGptService.build() + asyncio.run(chatgpt_service.update_chatgpt_models()) + logger.info("chatgpt models has been updated") diff --git a/bot_microservice/core/bot/repository.py b/bot_microservice/core/bot/repository.py index 08688a6..ccbc767 100644 --- a/bot_microservice/core/bot/repository.py +++ b/bot_microservice/core/bot/repository.py @@ -38,6 +38,18 @@ class ChatGPTRepository: async with self.db.session() as session: await session.execute(query) + async def delete_all_chatgpt_models(self) -> None: + query = delete(ChatGptModels) + async with self.db.session() as session: + await session.execute(query) + + async def bulk_insert_chatgpt_models(self, models_priority: list[dict[str, Any]]) -> None: + models = [ChatGptModels(**model_priority) for model_priority in models_priority] + + async with self.db.session() as session: + session.add_all(models) + await session.commit() + async def add_chatgpt_model(self, model: str, priority: int) -> dict[str, str | int]: query = ( insert(ChatGptModels) diff --git a/bot_microservice/core/bot/services.py b/bot_microservice/core/bot/services.py index 2963a39..4952597 100644 --- a/bot_microservice/core/bot/services.py +++ b/bot_microservice/core/bot/services.py @@ -14,7 +14,7 @@ from speech_recognition import ( UnknownValueError as SpeechRecognizerError, ) -from constants import AUDIO_SEGMENT_DURATION +from constants import AUDIO_SEGMENT_DURATION, ChatGptModelsEnum from core.auth.models.users import User from core.auth.repository import UserRepository from core.auth.services import UserService @@ -24,6 +24,77 @@ from infra.database.db_adapter import Database from settings.config import settings +@dataclass +class ChatGptService: + repository: ChatGPTRepository + user_service: UserService + + @classmethod + def build(cls) -> "ChatGptService": + db = Database(settings=settings) + repository = ChatGPTRepository(settings=settings, db=db) + user_repository = UserRepository(db=db) + user_service = UserService(repository=user_repository) + return ChatGptService(repository=repository, user_service=user_service) + + async def get_chatgpt_models(self) -> Sequence[ChatGptModels]: + return await self.repository.get_chatgpt_models() + + async def request_to_chatgpt(self, question: str | None) -> str: + question = question or "Привет!" + chatgpt_model = await self.get_current_chatgpt_model() + return await self.repository.ask_question(question=question, chatgpt_model=chatgpt_model) + + async def request_to_chatgpt_microservice(self, question: str) -> Response: + chatgpt_model = await self.get_current_chatgpt_model() + return await self.repository.request_to_chatgpt_microservice(question=question, chatgpt_model=chatgpt_model) + + async def get_current_chatgpt_model(self) -> str: + return await self.repository.get_current_chatgpt_model() + + async def change_chatgpt_model_priority(self, model_id: int, priority: int) -> None: + return await self.repository.change_chatgpt_model_priority(model_id=model_id, priority=priority) + + async def update_chatgpt_models(self) -> None: + await self.repository.delete_all_chatgpt_models() + models = ChatGptModelsEnum.base_models_priority() + await self.repository.bulk_insert_chatgpt_models(models) + + async def reset_all_chatgpt_models_priority(self) -> None: + return await self.repository.reset_all_chatgpt_models_priority() + + async def add_chatgpt_model(self, gpt_model: str, priority: int) -> dict[str, str | int]: + return await self.repository.add_chatgpt_model(model=gpt_model, priority=priority) + + async def delete_chatgpt_model(self, model_id: int) -> None: + return await self.repository.delete_chatgpt_model(model_id=model_id) + + async def get_or_create_bot_user( + self, + user_id: int, + email: str | None = None, + username: str | None = None, + first_name: str | None = None, + last_name: str | None = None, + ban_reason: str | None = None, + is_active: bool = True, + is_superuser: bool = False, + ) -> User: + return await self.user_service.get_or_create_user_by_id( + user_id=user_id, + email=email, + username=username, + first_name=first_name, + last_name=last_name, + ban_reason=ban_reason, + is_active=is_active, + is_superuser=is_superuser, + ) + + async def update_bot_user_message_count(self, user_id: int) -> None: + await self.user_service.update_user_message_count(user_id) + + class SpeechToTextService: def __init__(self, filename: str) -> None: self.filename = filename @@ -87,69 +158,3 @@ class SpeechToTextService: except SpeechRecognizerError as error: logger.error("error recognizing text with google", error=error) raise - - -@dataclass -class ChatGptService: - repository: ChatGPTRepository - user_service: UserService - - @classmethod - def build(cls) -> "ChatGptService": - db = Database(settings=settings) - repository = ChatGPTRepository(settings=settings, db=db) - user_repository = UserRepository(db=db) - user_service = UserService(repository=user_repository) - return ChatGptService(repository=repository, user_service=user_service) - - async def get_chatgpt_models(self) -> Sequence[ChatGptModels]: - return await self.repository.get_chatgpt_models() - - async def request_to_chatgpt(self, question: str | None) -> str: - question = question or "Привет!" - chatgpt_model = await self.get_current_chatgpt_model() - return await self.repository.ask_question(question=question, chatgpt_model=chatgpt_model) - - async def request_to_chatgpt_microservice(self, question: str) -> Response: - chatgpt_model = await self.get_current_chatgpt_model() - return await self.repository.request_to_chatgpt_microservice(question=question, chatgpt_model=chatgpt_model) - - async def get_current_chatgpt_model(self) -> str: - return await self.repository.get_current_chatgpt_model() - - async def change_chatgpt_model_priority(self, model_id: int, priority: int) -> None: - return await self.repository.change_chatgpt_model_priority(model_id=model_id, priority=priority) - - async def reset_all_chatgpt_models_priority(self) -> None: - return await self.repository.reset_all_chatgpt_models_priority() - - async def add_chatgpt_model(self, gpt_model: str, priority: int) -> dict[str, str | int]: - return await self.repository.add_chatgpt_model(model=gpt_model, priority=priority) - - async def delete_chatgpt_model(self, model_id: int) -> None: - return await self.repository.delete_chatgpt_model(model_id=model_id) - - async def get_or_create_bot_user( - self, - user_id: int, - email: str | None = None, - username: str | None = None, - first_name: str | None = None, - last_name: str | None = None, - ban_reason: str | None = None, - is_active: bool = True, - is_superuser: bool = False, - ) -> User: - return await self.user_service.get_or_create_user_by_id( - user_id=user_id, - email=email, - username=username, - first_name=first_name, - last_name=last_name, - ban_reason=ban_reason, - is_active=is_active, - is_superuser=is_superuser, - ) - - async def update_bot_user_message_count(self, user_id: int) -> None: - await self.user_service.update_user_message_count(user_id) diff --git a/bot_microservice/infra/admin.py b/bot_microservice/infra/admin.py index 3859f08..369793d 100644 --- a/bot_microservice/infra/admin.py +++ b/bot_microservice/infra/admin.py @@ -37,6 +37,7 @@ class UserAdmin(ModelView, model=User): User.is_active, User.ban_reason, "question_count", + "last_question_at", User.created_at, ] @@ -57,7 +58,14 @@ class UserAdmin(ModelView, model=User): ) ) .outerjoin(User.user_question_count) - .options(contains_eager(User.user_question_count).options(load_only(UserQuestionCount.question_count))) + .options( + contains_eager(User.user_question_count).options( + load_only( + UserQuestionCount.question_count, + UserQuestionCount.last_question_at, + ) + ) + ) ).order_by(desc(UserQuestionCount.question_count)) diff --git a/bot_microservice/infra/database/migrations/versions/0004_add_chatgpt_models.py b/bot_microservice/infra/database/migrations/versions/0004_add_chatgpt_models.py deleted file mode 100644 index 877b072..0000000 --- a/bot_microservice/infra/database/migrations/versions/0004_add_chatgpt_models.py +++ /dev/null @@ -1,43 +0,0 @@ -"""create chatgpt models - -Revision ID: 0004_add_chatgpt_models -Revises: 0003_create_user_question_count_table -Create Date: 2025-10-05 20:44:05.414977 - -""" -from loguru import logger -from sqlalchemy import select, text - -from constants import ChatGptModelsEnum -from core.bot.models.chatgpt import ChatGptModels -from infra.database.deps import get_sync_session - -# revision identifiers, used by Alembic. -revision = "0004_add_chatgpt_models" -down_revision = "0003_create_user_question_count_table" -branch_labels: str | None = None -depends_on: str | None = None - - -def upgrade() -> None: - with get_sync_session() as session: - query = select(ChatGptModels) - results = session.execute(query) - models = results.scalars().all() - - if models: - return - models = [] - for data in ChatGptModelsEnum.base_models_priority(): - models.append(ChatGptModels(**data)) - session.add_all(models) - session.commit() - - -def downgrade() -> None: - chatgpt_table_name = ChatGptModels.__tablename__ - with get_sync_session() as session: - # Truncate doesn't exists for SQLite - session.execute(text(f"""DELETE FROM {chatgpt_table_name}""")) # noqa: S608 - session.commit() - logger.info("chatgpt models table has been truncated", table=chatgpt_table_name) diff --git a/bot_microservice/infra/database/migrations/versions/0004_add_last_question_at.py b/bot_microservice/infra/database/migrations/versions/0004_add_last_question_at.py new file mode 100644 index 0000000..1daeb4b --- /dev/null +++ b/bot_microservice/infra/database/migrations/versions/0004_add_last_question_at.py @@ -0,0 +1,29 @@ +"""add_last_question_at + +Revision ID: 0004_add_last_question_at +Revises: 0003_create_user_question_count_table +Create Date: 2024-01-08 20:56:34.815976 + +""" +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision = "0004_add_last_question_at" +down_revision = "0003_create_user_question_count_table" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_index("ix_access_token_created_at", table_name="access_token") + op.add_column("user_question_count", sa.Column("last_question_at", sa.TIMESTAMP(timezone=True), nullable=False)) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column("user_question_count", "last_question_at") + op.create_index("ix_access_token_created_at", "access_token", ["created_at"], unique=False) + # ### end Alembic commands ### diff --git a/bot_microservice/tests/integration/bot/test_bot_updates.py b/bot_microservice/tests/integration/bot/test_bot_updates.py index 0a2f746..34dc32f 100644 --- a/bot_microservice/tests/integration/bot/test_bot_updates.py +++ b/bot_microservice/tests/integration/bot/test_bot_updates.py @@ -1,4 +1,5 @@ import asyncio +import datetime from unittest import mock import httpx @@ -414,6 +415,9 @@ async def test_ask_question_action_bot_user_not_exists( created_user_question_count = dbsession.query(UserQuestionCount).filter_by(user_id=user["id"]).one() assert created_user_question_count.question_count == 1 + assert created_user_question_count.last_question_at - datetime.datetime.now() < datetime.timedelta( # noqa: DTZ005 + seconds=2 + ) async def test_ask_question_action_bot_user_already_exists( @@ -466,6 +470,9 @@ async def test_ask_question_action_bot_user_already_exists( updated_user_question_count = dbsession.query(UserQuestionCount).filter_by(user_id=user["id"]).one() assert updated_user_question_count.question_count == existing_user_question_count + 1 + assert updated_user_question_count.last_question_at - datetime.datetime.now() < datetime.timedelta( # noqa: DTZ005 + seconds=2 + ) async def test_ask_question_action_user_is_banned( diff --git a/bot_microservice/tests/integration/bot/test_chatgpt_models.py b/bot_microservice/tests/integration/bot/test_chatgpt_models.py new file mode 100644 index 0000000..a6d9cb2 --- /dev/null +++ b/bot_microservice/tests/integration/bot/test_chatgpt_models.py @@ -0,0 +1,30 @@ +import pytest +from sqlalchemy.orm import Session + +from constants import ChatGptModelsEnum +from core.bot.models.chatgpt import ChatGptModels +from core.bot.services import ChatGptService + +pytestmark = [ + pytest.mark.asyncio, + pytest.mark.enable_socket, +] + + +async def test_models_update(dbsession: Session) -> None: + models = dbsession.query(ChatGptModels).all() + + assert len(models) == 0 + + chatgpt_service = ChatGptService.build() + + await chatgpt_service.update_chatgpt_models() + + models = dbsession.query(ChatGptModels).all() + + model_priorities = {model.model: model.priority for model in models} + + assert len(models) == len(ChatGptModelsEnum.base_models_priority()) + + for model_priority in ChatGptModelsEnum.base_models_priority(): + assert model_priorities[model_priority["model"]] == model_priority["priority"] diff --git a/bot_microservice/tests/integration/factories/user.py b/bot_microservice/tests/integration/factories/user.py index 5a117cf..f7510a3 100644 --- a/bot_microservice/tests/integration/factories/user.py +++ b/bot_microservice/tests/integration/factories/user.py @@ -34,6 +34,7 @@ class AccessTokenFactory(BaseModelFactory): class UserQuestionCountFactory(BaseModelFactory): user_id = factory.Sequence(lambda n: n + 1) question_count = factory.Faker("random_int") + last_question_at = factory.Faker("past_datetime") class Meta: model = UserQuestionCount diff --git a/poetry.lock b/poetry.lock index 39b4bbc..aa882f1 100644 --- a/poetry.lock +++ b/poetry.lock @@ -954,25 +954,6 @@ files = [ [package.dependencies] flake8 = ">3.0.0" -[[package]] -name = "flake8-noqa" -version = "1.4.0" -description = "Flake8 noqa comment validation" -optional = false -python-versions = ">=3.7" -files = [ - {file = "flake8-noqa-1.4.0.tar.gz", hash = "sha256:771765ab27d1efd157528379acd15131147f9ae578a72d17fb432ca197881243"}, - {file = "flake8_noqa-1.4.0-py3-none-any.whl", hash = "sha256:4465e16a19be433980f6f563d05540e2e54797eb11facb9feb50fed60624dc45"}, -] - -[package.dependencies] -flake8 = ">=3.8.0,<8.0" -typing-extensions = ">=3.7.4.2" - -[package.extras] -dev = ["flake8 (>=3.8.0,<6.0.0)", "flake8-annotations", "flake8-bandit", "flake8-bugbear", "flake8-commas", "flake8-comprehensions", "flake8-continuation", "flake8-datetimez", "flake8-docstrings", "flake8-import-order", "flake8-literal", "flake8-modern-annotations", "flake8-noqa", "flake8-polyfill", "flake8-pyproject", "flake8-requirements", "flake8-typechecking-import", "flake8-use-fstring", "mypy", "pep8-naming"] -test = ["flake8-docstrings"] - [[package]] name = "flake8-plugin-utils" version = "1.3.3" @@ -3732,4 +3713,4 @@ multidict = ">=4.0" [metadata] lock-version = "2.0" python-versions = "^3.12" -content-hash = "79650f1ee926621124c6b8cc3e3ddbf6c438ae15dee98f9bc9e1fddef07a8a70" +content-hash = "30a4b709eed24d6b55f78015994cdf1901d0d7279fefa86b37e9490e189a2c52" diff --git a/pyproject.toml b/pyproject.toml index 3370304..17e3d52 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "chat_gpt_bot" -version = "1.5.0" +version = "1.5.1" description = "Bot to integrated with Chat gpt" authors = ["Dmitry Afanasyev "] @@ -84,7 +84,6 @@ autoflake = "^2.2" flake8-aaa = "^0.17.0" flake8-variables-names = "^0.0.6" flake8-deprecated = "^2.2.1" -flake8-noqa = "^1.4" flake8-annotations-complexity = "^0.0.8" flake8-useless-assert = "^0.4.4" flake8-newspaper-style = "^0.4.3" diff --git a/scripts/start-bot.sh b/scripts/start-bot.sh index d338d1b..6106d30 100644 --- a/scripts/start-bot.sh +++ b/scripts/start-bot.sh @@ -4,9 +4,7 @@ set -e if [ -f shared/${DB_NAME:-chatgpt.db} ] then - alembic downgrade -1 && alembic upgrade "head" -else - alembic upgrade "head" + python3 core/bot/managment/update_gpt_models.py fi echo "starting the bot"