From 6ec8a3ce0a52f93d61538a0303e3326c373f45d6 Mon Sep 17 00:00:00 2001 From: grillazz Date: Sat, 26 Jul 2025 18:59:28 +0200 Subject: [PATCH 1/4] wip: add RotatingBytesLogger --- app/api/health.py | 5 ++- app/utils/logging.py | 72 ++++++++++++++++++++++++++++++++------------ 2 files changed, 54 insertions(+), 23 deletions(-) diff --git a/app/api/health.py b/app/api/health.py index 3a59010..04ff174 100644 --- a/app/api/health.py +++ b/app/api/health.py @@ -1,4 +1,3 @@ -import logging from typing import Annotated from fastapi import APIRouter, Depends, Query, Request, status @@ -34,7 +33,7 @@ async def redis_check(request: Request): try: redis_info = await redis_client.info() except Exception as e: - logging.error(f"Redis error: {e}") + await logger.aerror(f"Redis error: {e}") return redis_info @@ -88,7 +87,7 @@ async def smtp_check( "subject": subject, } - logger.info("Sending email with data: %s", email_data) + await logger.info("Sending email.", email_data=email_data) await run_in_threadpool( smtp.send_email, diff --git a/app/utils/logging.py b/app/utils/logging.py index 7ba940a..e153a8c 100644 --- a/app/utils/logging.py +++ b/app/utils/logging.py @@ -10,25 +10,60 @@ from whenever._whenever import Instant from app.utils.singleton import SingletonMetaNoArgs +class RotatingBytesLogger: + """Logger that respects RotatingFileHandler's rotation capabilities.""" -# TODO: merge this wrapper with the one in structlog under one hood of AppLogger -class BytesToTextIOWrapper: - def __init__(self, handler, encoding="utf-8"): + def __init__(self, handler): self.handler = handler - self.encoding = encoding - def write(self, b): - if isinstance(b, bytes): - self.handler.stream.write(b.decode(self.encoding)) - else: - self.handler.stream.write(b) - self.handler.flush() + def msg(self, message): + """Process a message and pass it through the handler's emit method.""" + if isinstance(message, bytes): + message = message.decode("utf-8") - def flush(self): - self.handler.flush() + # Create a log record that will trigger rotation checks + record = logging.LogRecord( + name="structlog", + level=logging.INFO, + pathname="", + lineno=0, + msg=message.rstrip("\n"), + args=(), + exc_info=None + ) - def close(self): - self.handler.close() + # Check if rotation is needed before emitting + if self.handler.shouldRollover(record): + self.handler.doRollover() + + # Emit the record through the handler + self.handler.emit(record) + + # Required methods to make it compatible with structlog + def debug(self, message): + self.msg(message) + + def info(self, message): + self.msg(message) + + def warning(self, message): + self.msg(message) + + def error(self, message): + self.msg(message) + + def critical(self, message): + self.msg(message) + + +class RotatingBytesLoggerFactory: + """Factory that creates loggers that respect file rotation.""" + + def __init__(self, handler): + self.handler = handler + + def __call__(self, *args, **kwargs): + return RotatingBytesLogger(self.handler) @define(slots=True) @@ -40,8 +75,7 @@ class AppStructLogger(metaclass=SingletonMetaNoArgs): _log_path = Path(f"{_log_date}_{os.getpid()}.log") _handler = RotatingFileHandler( filename=_log_path, - mode="a", - maxBytes=10 * 1024 * 1024, + maxBytes=1000, backupCount=5, encoding="utf-8" ) @@ -55,11 +89,9 @@ class AppStructLogger(metaclass=SingletonMetaNoArgs): structlog.processors.TimeStamper(fmt="iso", utc=True), structlog.processors.JSONRenderer(serializer=orjson.dumps), ], - logger_factory=structlog.BytesLoggerFactory( - file=BytesToTextIOWrapper(_handler) - ) + logger_factory=RotatingBytesLoggerFactory(_handler) ) self._logger = structlog.get_logger() def get_logger(self) -> structlog.BoundLogger: - return self._logger + return self._logger \ No newline at end of file From a99a0e780bef71e338283125dbaa84558a8fa40a Mon Sep 17 00:00:00 2001 From: grillazz Date: Sat, 26 Jul 2025 19:25:46 +0200 Subject: [PATCH 2/4] wip: async logging --- app/api/health.py | 2 +- app/api/stuff.py | 6 +++--- app/api/user.py | 2 +- app/database.py | 2 +- app/main.py | 2 +- app/models/base.py | 2 +- app/services/auth.py | 2 +- app/services/scheduler.py | 4 ++-- app/utils/logging.py | 2 +- 9 files changed, 12 insertions(+), 12 deletions(-) diff --git a/app/api/health.py b/app/api/health.py index 04ff174..a725f4f 100644 --- a/app/api/health.py +++ b/app/api/health.py @@ -87,7 +87,7 @@ async def smtp_check( "subject": subject, } - await logger.info("Sending email.", email_data=email_data) + await logger.ainfo("Sending email.", email_data=email_data) await run_in_threadpool( smtp.send_email, diff --git a/app/api/stuff.py b/app/api/stuff.py index 07b39d8..0f73b92 100644 --- a/app/api/stuff.py +++ b/app/api/stuff.py @@ -21,13 +21,13 @@ async def create_multi_stuff( db_session.add_all(stuff_instances) await db_session.commit() except SQLAlchemyError as ex: - logger.error(f"Error inserting instances of Stuff: {repr(ex)}") + await logger.aerror(f"Error inserting instances of Stuff: {repr(ex)}") raise HTTPException( status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail=repr(ex) ) from ex else: - logger.info( - f"{len(stuff_instances)} instances of Stuff inserted into database." + await logger.ainfo( + f"{len(stuff_instances)} Stuff instances inserted into the database." ) return True diff --git a/app/api/user.py b/app/api/user.py index 0d5b3bb..a22c7c6 100644 --- a/app/api/user.py +++ b/app/api/user.py @@ -18,7 +18,7 @@ router = APIRouter(prefix="/v1/user") async def create_user( payload: UserSchema, request: Request, db_session: AsyncSession = Depends(get_db) ): - logger.info(f"Creating user: {payload}") + await logger.ainfo(f"Creating user: {payload}") _user: User = User(**payload.model_dump()) await _user.save(db_session) diff --git a/app/database.py b/app/database.py index c900087..035691f 100644 --- a/app/database.py +++ b/app/database.py @@ -29,5 +29,5 @@ async def get_db() -> AsyncGenerator: try: yield session except Exception as e: - logger.error(f"Error getting database session: {e}") + await logger.aerror(f"Error getting database session: {e}") raise diff --git a/app/main.py b/app/main.py index 70ca866..f1c0ffd 100644 --- a/app/main.py +++ b/app/main.py @@ -30,7 +30,7 @@ async def lifespan(app: FastAPI): min_size=5, max_size=20, ) - logger.info("Postgres pool created", idle_size=app.postgres_pool.get_idle_size()) + await logger.ainfo("Postgres pool created", idle_size=app.postgres_pool.get_idle_size()) yield finally: await app.redis.close() diff --git a/app/models/base.py b/app/models/base.py index 6f114b4..f66a98f 100644 --- a/app/models/base.py +++ b/app/models/base.py @@ -30,7 +30,7 @@ class Base(DeclarativeBase): db_session.add(self) return await db_session.commit() except SQLAlchemyError as ex: - logger.error(f"Error inserting instance of {self}: {repr(ex)}") + await logger.aerror(f"Error inserting instance of {self}: {repr(ex)}") raise HTTPException( status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail=repr(ex) ) from ex diff --git a/app/services/auth.py b/app/services/auth.py index 3509f8a..23a4452 100644 --- a/app/services/auth.py +++ b/app/services/auth.py @@ -40,7 +40,7 @@ class AuthBearer(HTTPBearer): raise HTTPException( status_code=403, detail="Invalid token or expired token." ) - logger.info(f"Token verified: {credentials.credentials}") + await logger.ainfo(f"Token verified: {credentials.credentials}") return credentials.credentials diff --git a/app/services/scheduler.py b/app/services/scheduler.py index 31a5673..b290376 100644 --- a/app/services/scheduler.py +++ b/app/services/scheduler.py @@ -15,9 +15,9 @@ logger = AppLogger().get_logger() async def tick(): async with AsyncSessionFactory() as session: stmt = text("select 1;") - logger.info(f">>>> Be or not to be...{datetime.now()}") + await logger.ainfo(f">>>> Be or not to be...{datetime.now()}") result = await session.execute(stmt) - logger.info(f">>>> Result: {result.scalar()}") + await logger.ainfo(f">>>> Result: {result.scalar()}") return True diff --git a/app/utils/logging.py b/app/utils/logging.py index e153a8c..23af6fd 100644 --- a/app/utils/logging.py +++ b/app/utils/logging.py @@ -66,7 +66,7 @@ class RotatingBytesLoggerFactory: return RotatingBytesLogger(self.handler) -@define(slots=True) +@define class AppStructLogger(metaclass=SingletonMetaNoArgs): _logger: structlog.BoundLogger = field(init=False) From ffccf8fda09efb0808ad71754b917908bcce2c8a Mon Sep 17 00:00:00 2001 From: grillazz Date: Sat, 26 Jul 2025 19:28:56 +0200 Subject: [PATCH 3/4] lint code --- app/utils/logging.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/utils/logging.py b/app/utils/logging.py index 23af6fd..15873d3 100644 --- a/app/utils/logging.py +++ b/app/utils/logging.py @@ -10,6 +10,7 @@ from whenever._whenever import Instant from app.utils.singleton import SingletonMetaNoArgs + class RotatingBytesLogger: """Logger that respects RotatingFileHandler's rotation capabilities.""" @@ -94,4 +95,4 @@ class AppStructLogger(metaclass=SingletonMetaNoArgs): self._logger = structlog.get_logger() def get_logger(self) -> structlog.BoundLogger: - return self._logger \ No newline at end of file + return self._logger From 63859e8305ab07bca69174991815d85f8a1979f8 Mon Sep 17 00:00:00 2001 From: Ordinary Hobbit Date: Sat, 26 Jul 2025 19:33:13 +0200 Subject: [PATCH 4/4] Update app/utils/logging.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- app/utils/logging.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/utils/logging.py b/app/utils/logging.py index 15873d3..293efbe 100644 --- a/app/utils/logging.py +++ b/app/utils/logging.py @@ -76,7 +76,7 @@ class AppStructLogger(metaclass=SingletonMetaNoArgs): _log_path = Path(f"{_log_date}_{os.getpid()}.log") _handler = RotatingFileHandler( filename=_log_path, - maxBytes=1000, + maxBytes=10 * 1024 * 1024, # 10MB backupCount=5, encoding="utf-8" )