Compare commits

...

7 Commits

Author SHA1 Message Date
grillazz
353ef0da95 lint code 2025-07-27 20:14:51 +02:00
grillazz
289883cf2e Merge remote-tracking branch 'origin/198-add-simple-caching' into 198-add-simple-caching 2025-07-27 20:09:42 +02:00
grillazz
a8c645ad95 switch logger to rotoger 2025-07-27 20:09:30 +02:00
Ordinary Hobbit
63859e8305
Update app/utils/logging.py
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
2025-07-26 19:33:13 +02:00
grillazz
ffccf8fda0 lint code 2025-07-26 19:28:56 +02:00
grillazz
a99a0e780b wip: async logging 2025-07-26 19:25:46 +02:00
grillazz
6ec8a3ce0a wip: add RotatingBytesLogger 2025-07-26 18:59:28 +02:00
14 changed files with 710 additions and 655 deletions

View File

@ -56,8 +56,6 @@ COPY /templates/ templates/
COPY .env app/
COPY alembic.ini /panettone/alembic.ini
COPY /alembic/ /panettone/alembic/
COPY logging-uvicorn.json /panettone/logging-uvicorn.json
COPY logging-granian.json /panettone/logging-granian.json
COPY pyproject.toml /panettone/pyproject.toml
RUN python -V

View File

@ -1,12 +1,11 @@
import logging
from typing import Annotated
from fastapi import APIRouter, Depends, Query, Request, status
from pydantic import EmailStr
from rotoger import AppStructLogger
from starlette.concurrency import run_in_threadpool
from app.services.smtp import SMTPEmailService
from app.utils.logging import AppStructLogger
logger = AppStructLogger().get_logger()
@ -34,7 +33,7 @@ async def redis_check(request: Request):
try:
redis_info = await redis_client.info()
except Exception as e:
logging.error(f"Redis error: {e}")
await logger.aerror(f"Redis error: {e}")
return redis_info
@ -88,7 +87,7 @@ async def smtp_check(
"subject": subject,
}
logger.info("Sending email with data: %s", email_data)
await logger.ainfo("Sending email.", email_data=email_data)
await run_in_threadpool(
smtp.send_email,

View File

@ -2,9 +2,9 @@ from typing import Annotated
from fastapi import APIRouter, Depends, Form
from fastapi.responses import StreamingResponse
from rotoger import AppStructLogger
from app.services.llm import get_llm_service
from app.utils.logging import AppStructLogger
logger = AppStructLogger().get_logger()

View File

@ -1,11 +1,11 @@
from fastapi import APIRouter, Depends, HTTPException, Request, status
from rotoger import AppStructLogger
from sqlalchemy.exc import SQLAlchemyError
from sqlalchemy.ext.asyncio import AsyncSession
from app.database import get_db
from app.models.stuff import Stuff
from app.schemas.stuff import StuffResponse, StuffSchema
from app.utils.logging import AppStructLogger
logger = AppStructLogger().get_logger()
@ -21,13 +21,13 @@ async def create_multi_stuff(
db_session.add_all(stuff_instances)
await db_session.commit()
except SQLAlchemyError as ex:
logger.error(f"Error inserting instances of Stuff: {repr(ex)}")
await logger.aerror(f"Error inserting instances of Stuff: {repr(ex)}")
raise HTTPException(
status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail=repr(ex)
) from ex
else:
logger.info(
f"{len(stuff_instances)} instances of Stuff inserted into database."
await logger.ainfo(
f"{len(stuff_instances)} Stuff instances inserted into the database."
)
return True

View File

@ -1,13 +1,13 @@
from typing import Annotated
from fastapi import APIRouter, Depends, Form, HTTPException, Request, status
from rotoger import AppStructLogger
from sqlalchemy.ext.asyncio import AsyncSession
from app.database import get_db
from app.models.user import User
from app.schemas.user import TokenResponse, UserLogin, UserResponse, UserSchema
from app.services.auth import create_access_token
from app.utils.logging import AppStructLogger
logger = AppStructLogger().get_logger()
@ -18,7 +18,7 @@ router = APIRouter(prefix="/v1/user")
async def create_user(
payload: UserSchema, request: Request, db_session: AsyncSession = Depends(get_db)
):
logger.info(f"Creating user: {payload}")
await logger.ainfo(f"Creating user: {payload}")
_user: User = User(**payload.model_dump())
await _user.save(db_session)

View File

@ -1,9 +1,9 @@
from collections.abc import AsyncGenerator
from rotoger import AppStructLogger
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
from app.config import settings as global_settings
from app.utils.logging import AppStructLogger
logger = AppStructLogger().get_logger()
@ -29,5 +29,5 @@ async def get_db() -> AsyncGenerator:
try:
yield session
except Exception as e:
logger.error(f"Error getting database session: {e}")
await logger.aerror(f"Error getting database session: {e}")
raise

View File

@ -5,6 +5,7 @@ import asyncpg
from fastapi import Depends, FastAPI, Request
from fastapi.responses import HTMLResponse
from fastapi.templating import Jinja2Templates
from rotoger import AppStructLogger
from app.api.health import router as health_router
from app.api.ml import router as ml_router
@ -15,7 +16,6 @@ from app.api.user import router as user_router
from app.config import settings as global_settings
from app.redis import get_redis
from app.services.auth import AuthBearer
from app.utils.logging import AppStructLogger
logger = AppStructLogger().get_logger()
templates = Jinja2Templates(directory=Path(__file__).parent.parent / "templates")
@ -30,7 +30,7 @@ async def lifespan(app: FastAPI):
min_size=5,
max_size=20,
)
logger.info("Postgres pool created", idle_size=app.postgres_pool.get_idle_size())
await logger.ainfo("Postgres pool created", idle_size=app.postgres_pool.get_idle_size())
yield
finally:
await app.redis.close()

View File

@ -2,12 +2,11 @@ from typing import Any
from asyncpg import UniqueViolationError
from fastapi import HTTPException, status
from rotoger import AppStructLogger
from sqlalchemy.exc import IntegrityError, SQLAlchemyError
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy.orm import DeclarativeBase, declared_attr
from app.utils.logging import AppStructLogger
logger = AppStructLogger().get_logger()
@ -30,7 +29,7 @@ class Base(DeclarativeBase):
db_session.add(self)
return await db_session.commit()
except SQLAlchemyError as ex:
logger.error(f"Error inserting instance of {self}: {repr(ex)}")
await logger.aerror(f"Error inserting instance of {self}: {repr(ex)}")
raise HTTPException(
status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail=repr(ex)
) from ex

View File

@ -3,10 +3,10 @@ import time
import jwt
from fastapi import HTTPException, Request
from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer
from rotoger import AppStructLogger
from app.config import settings as global_settings
from app.models.user import User
from app.utils.logging import AppStructLogger
logger = AppStructLogger().get_logger()
@ -40,7 +40,7 @@ class AuthBearer(HTTPBearer):
raise HTTPException(
status_code=403, detail="Invalid token or expired token."
)
logger.info(f"Token verified: {credentials.credentials}")
await logger.ainfo(f"Token verified: {credentials.credentials}")
return credentials.credentials

View File

@ -15,9 +15,9 @@ logger = AppLogger().get_logger()
async def tick():
async with AsyncSessionFactory() as session:
stmt = text("select 1;")
logger.info(f">>>> Be or not to be...{datetime.now()}")
await logger.ainfo(f">>>> Be or not to be...{datetime.now()}")
result = await session.execute(stmt)
logger.info(f">>>> Result: {result.scalar()}")
await logger.ainfo(f">>>> Result: {result.scalar()}")
return True

View File

@ -5,9 +5,9 @@ from email.mime.text import MIMEText
from attrs import define, field
from fastapi.templating import Jinja2Templates
from pydantic import EmailStr
from rotoger import AppStructLogger
from app.config import settings as global_settings
from app.utils.logging import AppStructLogger
from app.utils.singleton import SingletonMetaNoArgs
logger = AppStructLogger().get_logger()

View File

@ -11,27 +11,63 @@ from whenever._whenever import Instant
from app.utils.singleton import SingletonMetaNoArgs
# TODO: merge this wrapper with the one in structlog under one hood of AppLogger
class BytesToTextIOWrapper:
def __init__(self, handler, encoding="utf-8"):
class RotatingBytesLogger:
"""Logger that respects RotatingFileHandler's rotation capabilities."""
def __init__(self, handler):
self.handler = handler
self.encoding = encoding
def write(self, b):
if isinstance(b, bytes):
self.handler.stream.write(b.decode(self.encoding))
else:
self.handler.stream.write(b)
self.handler.flush()
def msg(self, message):
"""Process a message and pass it through the handler's emit method."""
if isinstance(message, bytes):
message = message.decode("utf-8")
def flush(self):
self.handler.flush()
# Create a log record that will trigger rotation checks
record = logging.LogRecord(
name="structlog",
level=logging.INFO,
pathname="",
lineno=0,
msg=message.rstrip("\n"),
args=(),
exc_info=None
)
def close(self):
self.handler.close()
# Check if rotation is needed before emitting
if self.handler.shouldRollover(record):
self.handler.doRollover()
# Emit the record through the handler
self.handler.emit(record)
# Required methods to make it compatible with structlog
def debug(self, message):
self.msg(message)
def info(self, message):
self.msg(message)
def warning(self, message):
self.msg(message)
def error(self, message):
self.msg(message)
def critical(self, message):
self.msg(message)
@define(slots=True)
class RotatingBytesLoggerFactory:
"""Factory that creates loggers that respect file rotation."""
def __init__(self, handler):
self.handler = handler
def __call__(self, *args, **kwargs):
return RotatingBytesLogger(self.handler)
@define
class AppStructLogger(metaclass=SingletonMetaNoArgs):
_logger: structlog.BoundLogger = field(init=False)
@ -40,8 +76,7 @@ class AppStructLogger(metaclass=SingletonMetaNoArgs):
_log_path = Path(f"{_log_date}_{os.getpid()}.log")
_handler = RotatingFileHandler(
filename=_log_path,
mode="a",
maxBytes=10 * 1024 * 1024,
maxBytes=10 * 1024 * 1024, # 10MB
backupCount=5,
encoding="utf-8"
)
@ -55,9 +90,7 @@ class AppStructLogger(metaclass=SingletonMetaNoArgs):
structlog.processors.TimeStamper(fmt="iso", utc=True),
structlog.processors.JSONRenderer(serializer=orjson.dumps),
],
logger_factory=structlog.BytesLoggerFactory(
file=BytesToTextIOWrapper(_handler)
)
logger_factory=RotatingBytesLoggerFactory(_handler)
)
self._logger = structlog.get_logger()

View File

@ -29,8 +29,7 @@ dependencies = [
"polyfactory>=2.21.0",
"granian>=2.3.2",
"apscheduler[redis,sqlalchemy]>=4.0.0a6",
"structlog>=25.4.0",
"whenever>=0.8.5",
"rotoger",
]
[tool.uv]

1243
uv.lock generated

File diff suppressed because it is too large Load Diff