add user messages count action (#76)

* remove fastapi users dependency

* add user service to chatbot service

* add user save on bot info command

* add user model to admin

* fix tests
This commit is contained in:
Dmitry Afanasyev 2024-01-07 02:14:44 +03:00 committed by GitHub
parent fd9d38b5f0
commit 1e79c981c2
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
19 changed files with 811 additions and 798 deletions

View File

@ -150,7 +150,7 @@ alembic downgrade base
## Help article
[Следить за обновлениями этого репозитория](https://github.com/fantasy-peak/cpp-freegpt-webui)
[Следить за обновлениями этого репозитория](https://github.com/fantasy-peak/cpp-freegpt-webui/commits/main/)
## TODO
@ -161,7 +161,9 @@ alembic downgrade base
- [ ] and models rotation
- [x] add update model priority endpoint
- [x] add more tests for gpt model selection
- [ ] add authorisation for api
- [ ] add authorization for api
- [x] reformat conftest.py file
- [x] Add sentry
- [x] Add graylog integration and availability to log to file
- [x] add sentry
- [x] add graylog integration and availability to log to file
- [x] add user model
- [ ] add messages statistic

View File

@ -0,0 +1,16 @@
from fastapi import Depends
from api.deps import get_database
from core.auth.repository import UserRepository
from core.auth.services import UserService
from infra.database.db_adapter import Database
def get_user_repository(db: Database = Depends(get_database)) -> UserRepository:
return UserRepository(db=db)
def get_user_service(
user_repository: UserRepository = Depends(get_user_repository),
) -> UserService:
return UserService(repository=user_repository)

View File

@ -3,13 +3,13 @@ from starlette import status
from starlette.responses import JSONResponse, Response
from telegram import Update
from api.bot.deps import get_bot_queue, get_chatgpt_service, get_update_from_request
from api.bot.serializers import (
ChatGptModelSerializer,
ChatGptModelsPrioritySerializer,
GETChatGptModelsSerializer,
LightChatGptModel,
)
from api.deps import get_bot_queue, get_chatgpt_service, get_update_from_request
from core.bot.app import BotQueue
from core.bot.services import ChatGptService
from settings.config import settings

View File

@ -0,0 +1,42 @@
from fastapi import Depends
from starlette.requests import Request
from telegram import Update
from api.auth.deps import get_user_service
from api.deps import get_database
from core.auth.services import UserService
from core.bot.app import BotApplication, BotQueue
from core.bot.repository import ChatGPTRepository
from core.bot.services import ChatGptService
from infra.database.db_adapter import Database
from settings.config import AppSettings, get_settings
def get_bot_app(request: Request) -> BotApplication:
return request.app.state.bot_app
def get_bot_queue(request: Request) -> BotQueue:
return request.app.state.queue
async def get_update_from_request(request: Request, bot_app: BotApplication = Depends(get_bot_app)) -> Update | None:
data = await request.json()
return Update.de_json(data, bot_app.bot)
def get_chatgpt_repository(
db: Database = Depends(get_database), settings: AppSettings = Depends(get_settings)
) -> ChatGPTRepository:
return ChatGPTRepository(settings=settings, db=db)
def new_bot_queue(bot_app: BotApplication = Depends(get_bot_app)) -> BotQueue:
return BotQueue(bot_app=bot_app)
def get_chatgpt_service(
chatgpt_repository: ChatGPTRepository = Depends(get_chatgpt_repository),
user_service: UserService = Depends(get_user_service),
) -> ChatGptService:
return ChatGptService(repository=chatgpt_repository, user_service=user_service)

View File

@ -1,13 +1,7 @@
from fastapi import Depends
from fastapi_users_db_sqlalchemy import SQLAlchemyUserDatabase
from sqlalchemy.ext.asyncio import AsyncSession
from starlette.requests import Request
from telegram import Update
from core.auth.models.users import User
from core.bot.app import BotApplication, BotQueue
from core.bot.repository import ChatGPTRepository
from core.bot.services import ChatGptService
from infra.database.db_adapter import Database
from settings.config import AppSettings
@ -16,44 +10,9 @@ def get_settings(request: Request) -> AppSettings:
return request.app.state.settings
def get_bot_app(request: Request) -> BotApplication:
return request.app.state.bot_app
def get_bot_queue(request: Request) -> BotQueue:
return request.app.state.queue
def get_db_session(request: Request) -> AsyncSession:
return request.app.state.db_session_factory()
async def get_update_from_request(request: Request, bot_app: BotApplication = Depends(get_bot_app)) -> Update | None:
data = await request.json()
return Update.de_json(data, bot_app.bot)
def get_database(settings: AppSettings = Depends(get_settings)) -> Database:
return Database(settings=settings)
def get_chatgpt_repository(
db: Database = Depends(get_database), settings: AppSettings = Depends(get_settings)
) -> ChatGPTRepository:
return ChatGPTRepository(settings=settings, db=db)
def new_bot_queue(bot_app: BotApplication = Depends(get_bot_app)) -> BotQueue:
return BotQueue(bot_app=bot_app)
def get_chatgpt_service(
chatgpt_repository: ChatGPTRepository = Depends(get_chatgpt_repository),
) -> ChatGptService:
return ChatGptService(repository=chatgpt_repository)
async def get_user_db( # type: ignore[misc]
session: AsyncSession = Depends(get_db_session),
) -> SQLAlchemyUserDatabase: # type: ignore[type-arg]
yield SQLAlchemyUserDatabase(session, User)

View File

@ -3,7 +3,7 @@ from fastapi.responses import ORJSONResponse
from starlette import status
from starlette.responses import Response
from api.deps import get_chatgpt_service
from api.bot.deps import get_chatgpt_service
from api.exceptions import BaseAPIException
from constants import INVALID_GPT_REQUEST_MESSAGES
from core.bot.services import ChatGptService

View File

@ -0,0 +1,7 @@
from dataclasses import dataclass
@dataclass
class UserIsBannedDTO:
is_banned: bool = False
ban_reason: str | None = None

View File

@ -1,25 +1,76 @@
from fastapi_users_db_sqlalchemy import SQLAlchemyBaseUserTable
from fastapi_users_db_sqlalchemy.access_token import SQLAlchemyBaseAccessTokenTable
from sqlalchemy import INTEGER, VARCHAR, ForeignKey
from sqlalchemy.orm import Mapped, declared_attr, mapped_column
from datetime import datetime
from sqlalchemy import INTEGER, TIMESTAMP, VARCHAR, Boolean, ForeignKey, String
from sqlalchemy.orm import Mapped, mapped_column, relationship
from infra.database.base import Base
class User(SQLAlchemyBaseUserTable[Mapped[int]], Base):
class User(Base):
__tablename__ = "users" # type: ignore[assignment]
id: Mapped[int] = mapped_column(INTEGER, primary_key=True)
email: Mapped[str] = mapped_column(VARCHAR(length=320), unique=True, nullable=True) # type: ignore[assignment]
email: Mapped[str] = mapped_column(VARCHAR(length=255), unique=True, nullable=True)
username: Mapped[str] = mapped_column(VARCHAR(length=32), unique=True, index=True, nullable=False)
first_name: Mapped[str | None] = mapped_column(VARCHAR(length=32), nullable=True)
last_name: Mapped[str | None] = mapped_column(VARCHAR(length=32), nullable=True)
ban_reason: Mapped[str | None] = mapped_column(String(length=1024), nullable=True)
hashed_password: Mapped[str] = mapped_column(String(length=1024), nullable=False)
is_active: Mapped[bool] = mapped_column(Boolean, default=True, nullable=False)
is_superuser: Mapped[bool] = mapped_column(Boolean, default=False, nullable=False)
created_at: Mapped[datetime] = mapped_column(
TIMESTAMP(timezone=True), index=True, nullable=False, default=datetime.now
)
user_question_count: Mapped["UserQuestionCount"] = relationship(
"UserQuestionCount",
primaryjoin="UserQuestionCount.user_id == User.id",
backref="user",
lazy="selectin",
uselist=False,
)
@property
def question_count(self) -> int:
if self.user_question_count:
return self.user_question_count.question_count
return 0
@classmethod
def build(
cls,
id: int,
email: str | None = None,
username: str | None = None,
first_name: str | None = None,
last_name: str | None = None,
ban_reason: str | None = None,
hashed_password: str | None = None,
is_active: bool = True,
is_superuser: bool = False,
) -> "User":
username = username or str(id)
return User( # type: ignore[call-arg]
id=id,
email=email,
username=username,
first_name=first_name,
last_name=last_name,
ban_reason=ban_reason,
hashed_password=hashed_password,
is_active=is_active,
is_superuser=is_superuser,
)
class AccessToken(SQLAlchemyBaseAccessTokenTable[Mapped[int]], Base):
class AccessToken(Base):
__tablename__ = "access_token" # type: ignore[assignment]
@declared_attr
def user_id(cls) -> Mapped[int]:
return mapped_column(INTEGER, ForeignKey("users.id", ondelete="cascade"), nullable=False)
user_id = mapped_column(INTEGER, ForeignKey("users.id", ondelete="cascade"), nullable=False)
token: Mapped[str] = mapped_column(String(length=42), primary_key=True)
created_at: Mapped[datetime] = mapped_column(
TIMESTAMP(timezone=True), index=True, nullable=False, default=datetime.now
)
class UserQuestionCount(Base):

View File

@ -0,0 +1,78 @@
from dataclasses import dataclass
from sqlalchemy import select
from sqlalchemy.dialects.sqlite import insert
from sqlalchemy.orm import load_only
from core.auth.dto import UserIsBannedDTO
from core.auth.models.users import User, UserQuestionCount
from infra.database.db_adapter import Database
@dataclass
class UserRepository:
db: Database
async def create_user(
self,
id: int,
email: str | None,
username: str | None,
first_name: str | None,
last_name: str | None,
ban_reason: str | None,
hashed_password: str | None,
is_active: bool,
is_superuser: bool,
) -> User:
user = User.build(
id=id,
email=email,
username=username,
first_name=first_name,
last_name=last_name,
ban_reason=ban_reason,
hashed_password=hashed_password,
is_active=is_active,
is_superuser=is_superuser,
)
async with self.db.session() as session:
session.add(user)
await session.commit()
await session.refresh(user)
return user
async def get_user_by_id(self, user_id: int) -> User | None:
query = select(User).filter_by(id=user_id)
async with self.db.session() as session:
result = await session.execute(query)
return result.scalar()
async def check_user_is_banned(self, user_id: int) -> UserIsBannedDTO:
query = select(User).options(load_only(User.is_active, User.ban_reason)).filter_by(id=user_id)
async with self.db.session() as session:
result = await session.execute(query)
if user := result.scalar():
return UserIsBannedDTO(is_banned=not bool(user.is_active), ban_reason=user.ban_reason)
return UserIsBannedDTO()
async def update_user_message_count(self, user_id: int) -> None:
query = (
insert(UserQuestionCount)
.values({UserQuestionCount.user_id: user_id, UserQuestionCount.question_count: 1})
.on_conflict_do_update(
index_elements=[UserQuestionCount.user_id],
set_={
UserQuestionCount.get_real_column_name(
UserQuestionCount.question_count.key
): UserQuestionCount.question_count
+ 1
},
)
)
async with self.db.session() as session:
await session.execute(query)

View File

@ -0,0 +1,56 @@
import uuid
from dataclasses import dataclass
from core.auth.dto import UserIsBannedDTO
from core.auth.models.users import User
from core.auth.repository import UserRepository
from core.auth.utils import create_password_hash
from infra.database.db_adapter import Database
from settings.config import settings
@dataclass
class UserService:
repository: UserRepository
@classmethod
def build(cls) -> "UserService":
db = Database(settings=settings)
repository = UserRepository(db=db)
return UserService(repository=repository)
async def get_user_by_id(self, user_id: int) -> User | None:
return await self.repository.get_user_by_id(user_id)
async def get_or_create_user_by_id(
self,
user_id: int,
hashed_password: str | None = None,
email: str | None = None,
username: str | None = None,
first_name: str | None = None,
last_name: str | None = None,
ban_reason: str | None = None,
is_active: bool = True,
is_superuser: bool = False,
) -> User:
hashed_password = hashed_password or create_password_hash(uuid.uuid4().hex)
if not (user := await self.repository.get_user_by_id(user_id=user_id)):
user = await self.repository.create_user(
id=user_id,
email=email,
username=username,
first_name=first_name,
last_name=last_name,
ban_reason=ban_reason,
hashed_password=hashed_password,
is_active=is_active,
is_superuser=is_superuser,
)
return user
async def update_user_message_count(self, user_id: int) -> None:
await self.repository.update_user_message_count(user_id)
async def check_user_is_banned(self, user_id: int) -> UserIsBannedDTO:
return await self.repository.check_user_is_banned(user_id)

View File

@ -0,0 +1,9 @@
import hashlib
from settings.config import settings
def create_password_hash(password: str) -> str:
if not settings.SALT:
return password
return hashlib.sha256((password + settings.SALT.get_secret_value()).encode()).hexdigest()

View File

@ -92,6 +92,11 @@ async def ask_question(update: Update, context: ContextTypes.DEFAULT_TYPE) -> No
if not update.message:
return
if not update.effective_user:
logger.error('no effective user', update=update, context=context)
await update.message.reply_text("Бот не смог определить пользователя. :(\nОб ошибке уже сообщено.")
return
await update.message.reply_text(
f"Ответ в среднем занимает 10-15 секунд.\n"
f"- Список команд: /{BotCommands.help}\n"
@ -100,8 +105,16 @@ async def ask_question(update: Update, context: ContextTypes.DEFAULT_TYPE) -> No
chatgpt_service = ChatGptService.build()
logger.warning("question asked", user=update.message.from_user, question=update.message.text)
answer = await chatgpt_service.request_to_chatgpt(question=update.message.text)
await update.message.reply_text(answer)
answer, user = await asyncio.gather(
chatgpt_service.request_to_chatgpt(question=update.message.text),
chatgpt_service.get_or_create_bot_user(
user_id=update.effective_user.id,
username=update.effective_user.username,
first_name=update.effective_user.first_name,
last_name=update.effective_user.last_name,
),
)
await asyncio.gather(update.message.reply_text(answer), chatgpt_service.update_bot_user_message_count(user.id))
async def voice_recognize(update: Update, context: ContextTypes.DEFAULT_TYPE) -> None:

View File

@ -15,6 +15,9 @@ from speech_recognition import (
)
from constants import AUDIO_SEGMENT_DURATION
from core.auth.models.users import User
from core.auth.repository import UserRepository
from core.auth.services import UserService
from core.bot.models.chatgpt import ChatGptModels
from core.bot.repository import ChatGPTRepository
from infra.database.db_adapter import Database
@ -89,6 +92,15 @@ class SpeechToTextService:
@dataclass
class ChatGptService:
repository: ChatGPTRepository
user_service: UserService
@classmethod
def build(cls) -> "ChatGptService":
db = Database(settings=settings)
repository = ChatGPTRepository(settings=settings, db=db)
user_repository = UserRepository(db=db)
user_service = UserService(repository=user_repository)
return ChatGptService(repository=repository, user_service=user_service)
async def get_chatgpt_models(self) -> Sequence[ChatGptModels]:
return await self.repository.get_chatgpt_models()
@ -117,8 +129,27 @@ class ChatGptService:
async def delete_chatgpt_model(self, model_id: int) -> None:
return await self.repository.delete_chatgpt_model(model_id=model_id)
@classmethod
def build(cls) -> "ChatGptService":
db = Database(settings=settings)
repository = ChatGPTRepository(settings=settings, db=db)
return ChatGptService(repository=repository)
async def get_or_create_bot_user(
self,
user_id: int,
email: str | None = None,
username: str | None = None,
first_name: str | None = None,
last_name: str | None = None,
ban_reason: str | None = None,
is_active: bool = True,
is_superuser: bool = False,
) -> User:
return await self.user_service.get_or_create_user_by_id(
user_id=user_id,
email=email,
username=username,
first_name=first_name,
last_name=last_name,
ban_reason=ban_reason,
is_active=is_active,
is_superuser=is_superuser,
)
async def update_bot_user_message_count(self, user_id: int) -> None:
await self.user_service.update_user_message_count(user_id)

View File

@ -2,6 +2,7 @@ from typing import TYPE_CHECKING
from sqladmin import Admin, ModelView
from core.auth.models.users import User
from core.bot.models.chatgpt import ChatGptModels
from core.utils import build_uri
from settings.config import settings
@ -20,6 +21,22 @@ class ChatGptAdmin(ModelView, model=ChatGptModels):
can_delete = False
class UserAdmin(ModelView, model=User):
column_list = [
User.id,
User.username,
User.first_name,
User.last_name,
User.is_active,
User.ban_reason,
"question_count",
User.created_at,
]
column_sortable_list = [User.created_at]
column_default_sort = ("created_at", True)
form_widget_args = {"created_at": {"readonly": True}}
def create_admin(application: "Application") -> Admin:
admin = Admin(
title="Chat GPT admin",
@ -29,4 +46,5 @@ def create_admin(application: "Application") -> Admin:
authentication_backend=None,
)
admin.add_view(ChatGptAdmin)
admin.add_view(UserAdmin)
return admin

View File

@ -5,14 +5,15 @@ Revises: 0001_create_chatgpt_table
Create Date: 2023-11-28 00:58:01.984654
"""
import hashlib
from datetime import datetime
import fastapi_users_db_sqlalchemy
import sqlalchemy as sa
from alembic import op
from sqlalchemy import TIMESTAMP
from sqlalchemy.dialects.sqlite import insert
from core.auth.models.users import User
from core.auth.utils import create_password_hash
from infra.database.deps import get_sync_session
from settings.config import settings
@ -28,12 +29,15 @@ def upgrade() -> None:
op.create_table(
"users",
sa.Column("id", sa.INTEGER(), nullable=False),
sa.Column("email", sa.VARCHAR(length=320), nullable=True),
sa.Column("email", sa.VARCHAR(length=255), nullable=True),
sa.Column("username", sa.VARCHAR(length=32), nullable=False),
sa.Column("first_name", sa.VARCHAR(length=32), nullable=True),
sa.Column("last_name", sa.VARCHAR(length=32), nullable=True),
sa.Column("hashed_password", sa.String(length=1024), nullable=False),
sa.Column("is_active", sa.Boolean(), nullable=False),
sa.Column("is_superuser", sa.Boolean(), nullable=False),
sa.Column("is_verified", sa.Boolean(), nullable=False),
sa.Column("is_active", sa.Boolean(), nullable=False, default=True),
sa.Column("is_superuser", sa.Boolean(), nullable=False, default=False),
sa.Column("ban_reason", sa.String(length=1024), nullable=True),
sa.Column("created_at", TIMESTAMP(timezone=True), nullable=False, default=datetime.now),
sa.PrimaryKeyConstraint("id"),
sa.UniqueConstraint("email"),
)
@ -41,8 +45,8 @@ def upgrade() -> None:
op.create_table(
"access_token",
sa.Column("user_id", sa.INTEGER(), nullable=False),
sa.Column("token", sa.String(length=43), nullable=False),
sa.Column("created_at", fastapi_users_db_sqlalchemy.generics.TIMESTAMPAware(timezone=True), nullable=False),
sa.Column("token", sa.String(length=42), nullable=False),
sa.Column("created_at", TIMESTAMP(timezone=True), nullable=False, default=datetime.now),
sa.ForeignKeyConstraint(["user_id"], ["users.id"], ondelete="cascade"),
sa.PrimaryKeyConstraint("token"),
)
@ -53,7 +57,7 @@ def upgrade() -> None:
if not all([username, password, salt]):
return
with get_sync_session() as session:
hashed_password = hashlib.sha256((password.get_secret_value() + salt.get_secret_value()).encode()).hexdigest()
hashed_password = create_password_hash(password.get_secret_value())
query = insert(User).values({"username": username, "hashed_password": hashed_password})
session.execute(query)
session.commit()

View File

@ -61,7 +61,4 @@ RUN chmod +x ./start-bot.sh
COPY --from=compile-image /app/.venv /app/.venv
ENV PATH="/app/.venv/bin:$PATH"
# workarroud fo python3.12 and setuptools not found for fastapi users
RUN pip3 uninstall -y setuptools && pip3 install --upgrade setuptools wheel
USER ${USER}

View File

@ -54,7 +54,7 @@ services:
- "8858"
caddy:
image: "caddy:2.7.5"
image: "caddy:2.7.6"
container_name: "chatgpt_caddy_service"
hostname: "caddy_service"
restart: unless-stopped

1160
poetry.lock generated

File diff suppressed because it is too large Load Diff

View File

@ -11,7 +11,7 @@ build-backend = "poetry.core.masonry.api"
[tool.poetry.dependencies]
python = "^3.12"
fastapi = "^0.105"
fastapi = "^0.108"
python-telegram-bot = {version = "^20.6", extras=["ext"]}
python-dotenv = "^1.0"
python-dateutil = "*"
@ -21,7 +21,7 @@ loguru = "^0.7"
pydantic = "^2.5"
pydantic-settings = "^2.1"
gunicorn = "^21.2"
uvicorn = "^0.24"
uvicorn = "^0.25"
wheel = "^0.42"
orjson = "^3.9"
sentry-sdk = "^1.39"
@ -33,12 +33,12 @@ yarl = "^1.9"
sqlalchemy = {version = "^2.0", extras=["mypy"]}
alembic = "^1.13"
sqladmin = {version = "^0.16", extras=["full"]}
fastapi-users = {version = "^12.1.2", extras=["sqlalchemy"]}
pydub = {git = "https://github.com/jiaaro/pydub.git"}
types-pytz = "^2023.3.1.1"
[tool.poetry.dev-dependencies]
ipython = "^8.18"
ipython = "^8.19"
factory-boy = "^3.3"
Faker = "^20"
@ -53,7 +53,7 @@ pyupgrade = "^3.10"
isort = "^5.12"
black = "^23.12"
mypy = "^1.7"
mypy = "^1.8"
types-PyMySQL = "^1.0"
types-python-dateutil = "^2.8"
@ -78,7 +78,7 @@ pytest-socket = "^0.6"
assertpy = "^1.1"
respx = "^0.20"
coverage = "^7.3"
coverage = "^7.4"
autoflake = "^2.2"
flake8-aaa = "^0.17.0"