database refactor and remove redundant code

This commit is contained in:
Jakub Miazek 2022-09-27 12:54:37 +02:00
parent 9b1938c450
commit 307feeeade
20 changed files with 9 additions and 729 deletions

View File

@ -1,8 +1,11 @@
from asyncio import current_task
from typing import AsyncGenerator
from fastapi import HTTPException
from fastapi.encoders import jsonable_encoder
from sqlalchemy.exc import SQLAlchemyError
from sqlalchemy.ext.asyncio import AsyncSession, create_async_engine
from sqlalchemy.ext.asyncio import AsyncSession, create_async_engine, async_scoped_session
from sqlalchemy.orm import sessionmaker
from app import config
@ -14,16 +17,18 @@ engine = create_async_engine(
url,
future=True,
echo=True,
json_serializer=jsonable_encoder,
)
# expire_on_commit=False will prevent attributes from being expired
# after commit.
async_session = sessionmaker(engine, expire_on_commit=False, class_=AsyncSession)
async_session_factory = sessionmaker(engine, expire_on_commit=False, class_=AsyncSession)
AsyncScopedSession = async_scoped_session(async_session_factory, scopefunc=current_task)
# Dependency
async def get_db() -> AsyncGenerator:
async with async_session() as session:
async with async_session_factory() as session:
try:
yield session
await session.commit()
@ -35,3 +40,4 @@ async def get_db() -> AsyncGenerator:
raise http_ex
finally:
await session.close()

View File

View File

@ -1,40 +0,0 @@
from fastapi import APIRouter, Depends, status
from sqlalchemy.ext.asyncio import AsyncSession
from app.database import get_db
from app.models.nonsense import Nonsense
from app.schemas.nnonsense import NonsenseResponse, NonsenseSchema
router = APIRouter(prefix="/v1/nonsense")
@router.post("/", status_code=status.HTTP_201_CREATED, response_model=NonsenseResponse)
async def create_nonsense(payload: NonsenseSchema, db_session: AsyncSession = Depends(get_db)):
nonsense = Nonsense(**payload.dict())
await nonsense.save(db_session)
return nonsense
@router.get("/", response_model=NonsenseResponse)
async def find_nonsense(
name: str,
db_session: AsyncSession = Depends(get_db),
):
return await Nonsense.find(db_session, name)
@router.delete("/")
async def delete_nonsense(name: str, db_session: AsyncSession = Depends(get_db)):
nonsense = await Nonsense.find(db_session, name)
return await nonsense.delete(nonsense, db_session)
@router.patch("/", response_model=NonsenseResponse)
async def update_nonsense(
payload: NonsenseSchema,
name: str,
db_session: AsyncSession = Depends(get_db),
):
nonsense = await Nonsense.find(db_session, name)
await nonsense.update(db_session, **payload.dict())
return nonsense

View File

@ -1,15 +0,0 @@
from fastapi import APIRouter, Depends, status
from sqlalchemy.ext.asyncio import AsyncSession
from app.database import get_db
from app.models.shakespeare import Paragraph
router = APIRouter(prefix="/v1/shakespeare")
@router.get("/",)
async def find_paragraph(
character: str,
db_session: AsyncSession = Depends(get_db),
):
return await Paragraph.find(db_session=db_session, character=character)

View File

@ -1,58 +0,0 @@
from fastapi import APIRouter, Depends, HTTPException, status
from sqlalchemy.exc import SQLAlchemyError
from sqlalchemy.ext.asyncio import AsyncSession
from app.database import get_db
from app.models.stuff import Stuff
from app.schemas.stuff import StuffResponse, StuffSchema
from app.utils import get_logger
router = APIRouter(prefix="/v1/stuff")
logger = get_logger(__name__)
@router.post("/add_many", status_code=status.HTTP_201_CREATED)
async def create_multi_stuff(payload: list[StuffSchema], db_session: AsyncSession = Depends(get_db)):
try:
stuff_instances = [Stuff(name=stuf.name, description=stuf.description) for stuf in payload]
db_session.add_all(stuff_instances)
await db_session.commit()
except SQLAlchemyError as ex:
# logger.exception(ex)
raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail=repr(ex))
else:
logger.info(f"{len(stuff_instances)} instances of Stuff inserted into database.")
return True
@router.post("", status_code=status.HTTP_201_CREATED, response_model=StuffResponse)
async def create_stuff(payload: StuffSchema, db_session: AsyncSession = Depends(get_db)):
stuff = Stuff(name=payload.name, description=payload.description)
await stuff.save(db_session)
return stuff
@router.get("/{name}", response_model=StuffResponse)
async def find_stuff(
name: str,
db_session: AsyncSession = Depends(get_db),
):
return await Stuff.find(db_session, name)
@router.delete("/{name}")
async def delete_stuff(name: str, db_session: AsyncSession = Depends(get_db)):
stuff = await Stuff.find(db_session, name)
return await Stuff.delete(stuff, db_session)
@router.patch("/{name}", response_model=StuffResponse)
async def update_stuff(
payload: StuffSchema,
name: str,
db_session: AsyncSession = Depends(get_db),
):
stuff = await Stuff.find(db_session, name)
await stuff.update(db_session, **payload.dict())
return stuff

View File

@ -1,44 +0,0 @@
import os
from functools import lru_cache
from pydantic import BaseSettings
from app.utils import get_logger
logger = get_logger(__name__)
class Settings(BaseSettings):
"""
BaseSettings, from Pydantic, validates the data so that when we create an instance of Settings,
environment and testing will have types of str and bool, respectively.
Parameters:
pg_user (str):
pg_pass (str):
pg_database: (str):
pg_test_database: (str):
asyncpg_url: AnyUrl:
asyncpg_test_url: AnyUrl:
Returns:
instance of Settings
"""
pg_user: str = os.getenv("SQL_USER", "")
pg_pass: str = os.getenv("POSTGRES_PASSWORD", "")
pg_host: str = os.getenv("SQL_HOST", "")
pg_database: str = os.getenv("SQL_DB", "")
asyncpg_url: str = f"postgresql+asyncpg://{pg_user}:{pg_pass}@{pg_host}:5432/{pg_database}"
jwt_secret_key: str = os.getenv("SECRET_KEY", "")
jwt_algorithm: str = os.getenv("ALGORITHM", "")
jwt_access_toke_expire_minutes: int = os.getenv("ACCESS_TOKEN_EXPIRE_MINUTES", 1)
@lru_cache()
def get_settings():
logger.info("Loading config settings from the environment...")
return Settings()

View File

@ -1,37 +0,0 @@
from typing import AsyncGenerator
from fastapi import HTTPException
from sqlalchemy.exc import SQLAlchemyError
from sqlalchemy.ext.asyncio import AsyncSession, create_async_engine
from sqlalchemy.orm import sessionmaker
from app import config
global_settings = config.get_settings()
url = global_settings.asyncpg_url
engine = create_async_engine(
url,
future=True,
echo=True,
)
# expire_on_commit=False will prevent attributes from being expired
# after commit.
async_session = sessionmaker(engine, expire_on_commit=False, class_=AsyncSession)
# Dependency
async def get_db() -> AsyncGenerator:
async with async_session() as session:
try:
yield session
await session.commit()
except SQLAlchemyError as sql_ex:
await session.rollback()
raise sql_ex
except HTTPException as http_ex:
await session.rollback()
raise http_ex
finally:
await session.close()

View File

@ -1,59 +0,0 @@
from fastapi import HTTPException, status
class BadRequestHTTPException(HTTPException):
def __init__(self, msg: str):
super().__init__(
status_code=status.HTTP_400_BAD_REQUEST,
detail=msg if msg else "Bad request",
)
class AuthFailedHTTPException(HTTPException):
def __init__(self):
super().__init__(
status_code=status.HTTP_401_UNAUTHORIZED,
detail="Not authenticated",
headers={"WWW-Authenticate": "Bearer"},
)
class AuthTokenExpiredHTTPException(HTTPException):
def __init__(self):
super().__init__(
status_code=status.HTTP_401_UNAUTHORIZED,
detail="Expired token",
headers={"WWW-Authenticate": "Bearer"},
)
class ForbiddenHTTPException(HTTPException):
def __init__(self, msg: str):
super().__init__(
status_code=status.HTTP_403_FORBIDDEN,
detail=msg if msg else "Requested resource is forbidden",
)
class NotFoundHTTPException(HTTPException):
def __init__(self, msg: str):
super().__init__(
status_code=status.HTTP_404_NOT_FOUND,
detail=msg if msg else "Requested resource is not found",
)
class ConflictHTTPException(HTTPException):
def __init__(self, msg: str):
super().__init__(
status_code=status.HTTP_409_CONFLICT,
detail=msg if msg else "Conflicting resource request",
)
class ServiceNotAvailableHTTPException(HTTPException):
def __init__(self, msg: str):
super().__init__(
status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
detail=msg if msg else "Service not available",
)

View File

@ -1,24 +0,0 @@
from fastapi import FastAPI
from app.api.nonsense import router as nonsense_router
from app.api.stuff import router as stuff_router
from app.api.shakespeare import router as shakespeare_router
from app.utils import get_logger
logger = get_logger(__name__)
app = FastAPI(title="Stuff And Nonsense API", version="0.4")
app.include_router(stuff_router)
app.include_router(nonsense_router)
app.include_router(shakespeare_router)
@app.on_event("startup")
async def startup_event():
logger.info("Starting up...")
@app.on_event("shutdown")
async def shutdown_event():
logger.info("Shutting down...")

View File

@ -1,4 +0,0 @@
# for Alembic and unit tests
from app.models.stuff import * # noqa
from app.models.nonsense import * # noqa
from app.models.shakespeare import * # noqa

View File

@ -1,64 +0,0 @@
from typing import Any
from fastapi import HTTPException, status
from sqlalchemy.exc import SQLAlchemyError
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy.ext.declarative import as_declarative, declared_attr
@as_declarative()
class BaseReadOnly:
id: Any
__name__: str
# Generate __tablename__ automatically
@declared_attr
def __tablename__(cls) -> str:
return cls.__name__.lower()
@as_declarative()
class Base:
id: Any
__name__: str
# Generate __tablename__ automatically
@declared_attr
def __tablename__(cls) -> str:
return cls.__name__.lower()
async def save(self, db_session: AsyncSession):
"""
:param db_session:
:return:
"""
try:
db_session.add(self)
return await db_session.commit()
except SQLAlchemyError as ex:
raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail=repr(ex))
async def delete(self, db_session: AsyncSession):
"""
:param db_session:
:return:
"""
try:
await db_session.delete(self)
await db_session.commit()
return True
except SQLAlchemyError as ex:
raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail=repr(ex))
async def update(self, db_session: AsyncSession, **kwargs):
"""
:param db_session:
:param kwargs:
:return:
"""
for k, v in kwargs.items():
setattr(self, k, v)
await self.save(db_session)

View File

@ -1,42 +0,0 @@
import uuid
from fastapi import HTTPException, status
from sqlalchemy import Column, String, select
from sqlalchemy.dialects.postgresql import UUID
from sqlalchemy.ext.asyncio import AsyncSession
from app.models.base import Base
class Nonsense(Base):
__tablename__ = "nonsense"
__table_args__ = (
{"schema": "happy_hog"},
)
id = Column(UUID(as_uuid=True), unique=True, default=uuid.uuid4, autoincrement=True)
name = Column(String, nullable=False, primary_key=True, unique=True)
description = Column(String, nullable=False)
def __init__(self, name: str, description: str):
self.name = name
self.description = description
@classmethod
async def find(cls, db_session: AsyncSession, name: str):
"""
:param db_session:
:param name:
:return:
"""
stmt = select(cls).where(cls.name == name)
result = await db_session.execute(stmt)
instance = result.scalars().first()
if instance is None:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail={"Record not found": f"There is no record for requested name value : {name}"},
)
else:
return instance

View File

@ -1,134 +0,0 @@
from sqlalchemy import (
Column,
ForeignKey,
ForeignKeyConstraint,
Integer,
PrimaryKeyConstraint,
String,
Table,
Text,
UniqueConstraint,
select,
)
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy.orm import relationship
from app.models.base import Base
metadata = Base.metadata
class Character(Base):
__tablename__ = "character"
__table_args__ = (PrimaryKeyConstraint("id", name="character_pkey"), {"schema": "shakespeare"})
id = Column(String(32))
name = Column(String(64), nullable=False)
speech_count = Column(Integer, nullable=False)
abbrev = Column(String(32))
description = Column(String(2056))
work = relationship("Work", secondary="shakespeare.character_work", back_populates="character")
paragraph = relationship("Paragraph", back_populates="character")
class Wordform(Base):
__tablename__ = "wordform"
__table_args__ = (PrimaryKeyConstraint("id", name="wordform_pkey"), {"schema": "shakespeare"})
id = Column(Integer)
plain_text = Column(String(64), nullable=False)
phonetic_text = Column(String(64), nullable=False)
stem_text = Column(String(64), nullable=False)
occurences = Column(Integer, nullable=False)
class Work(Base):
__tablename__ = "work"
__table_args__ = (PrimaryKeyConstraint("id", name="work_pkey"), {"schema": "shakespeare"})
id = Column(String(32))
title = Column(String(32), nullable=False)
long_title = Column(String(64), nullable=False)
year = Column(Integer, nullable=False)
genre_type = Column(String(1), nullable=False)
source = Column(String(16), nullable=False)
total_words = Column(Integer, nullable=False)
total_paragraphs = Column(Integer, nullable=False)
notes = Column(Text)
character = relationship("Character", secondary="shakespeare.character_work", back_populates="work")
chapter = relationship("Chapter", back_populates="work")
paragraph = relationship("Paragraph", back_populates="work")
class Chapter(Base):
__tablename__ = "chapter"
__table_args__ = (
ForeignKeyConstraint(["work_id"], ["shakespeare.work.id"], name="chapter_work_id_fkey"),
PrimaryKeyConstraint("id", name="chapter_pkey"),
UniqueConstraint(
"work_id", "section_number", "chapter_number", name="chapter_work_id_section_number_chapter_number_key"
),
{"schema": "shakespeare"},
)
id = Column(Integer)
work_id = Column(ForeignKey("shakespeare.work.id"), nullable=False)
section_number = Column(Integer, nullable=False)
chapter_number = Column(Integer, nullable=False)
description = Column(String(256), nullable=False)
work = relationship("Work", back_populates="chapter")
paragraph = relationship("Paragraph", back_populates="chapter")
t_character_work = Table(
"character_work",
metadata,
Column("character_id", ForeignKey("shakespeare.character.id"), nullable=False),
Column("work_id", ForeignKey("shakespeare.work.id"), nullable=False),
ForeignKeyConstraint(["character_id"], ["shakespeare.character.id"], name="character_work_character_id_fkey"),
ForeignKeyConstraint(["work_id"], ["shakespeare.work.id"], name="character_work_work_id_fkey"),
PrimaryKeyConstraint("character_id", "work_id", name="character_work_pkey"),
schema="shakespeare",
)
class Paragraph(Base):
__tablename__ = "paragraph"
__table_args__ = (
ForeignKeyConstraint(["character_id"], ["shakespeare.character.id"], name="paragraph_character_id_fkey"),
ForeignKeyConstraint(
["work_id", "section_number", "chapter_number"],
["shakespeare.chapter.work_id", "shakespeare.chapter.section_number", "shakespeare.chapter.chapter_number"],
name="paragraph_chapter_fkey",
),
ForeignKeyConstraint(["work_id"], ["shakespeare.work.id"], name="paragraph_work_id_fkey"),
PrimaryKeyConstraint("id", name="paragraph_pkey"),
{"schema": "shakespeare"},
)
id = Column(Integer)
work_id = Column(ForeignKey("shakespeare.work.id"), nullable=False)
paragraph_num = Column(Integer, nullable=False)
character_id = Column(ForeignKey("shakespeare.character.id"), nullable=False)
plain_text = Column(Text, nullable=False)
phonetic_text = Column(Text, nullable=False)
stem_text = Column(Text, nullable=False)
paragraph_type = Column(String(1), nullable=False)
section_number = Column(Integer, nullable=False)
chapter_number = Column(Integer, nullable=False)
char_count = Column(Integer, nullable=False)
word_count = Column(Integer, nullable=False)
character = relationship("Character", back_populates="paragraph", lazy="selectin")
chapter = relationship("Chapter", back_populates="paragraph", lazy="selectin")
work = relationship("Work", back_populates="paragraph", lazy="selectin")
@classmethod
async def find(cls, db_session: AsyncSession, character: str):
stmt = select(cls).join(Character).join(Chapter).join(Work).where(Character.name == character)
result = await db_session.execute(stmt)
instance = result.scalars().all()
return instance

View File

@ -1,41 +0,0 @@
import uuid
from fastapi import HTTPException, status
from sqlalchemy import Column, String, select
from sqlalchemy.dialects.postgresql import UUID
from sqlalchemy.ext.asyncio import AsyncSession
from app.models.base import Base
class Stuff(Base):
__tablename__ = "stuff"
__table_args__ = (
{"schema": "happy_hog"},
)
id = Column(UUID(as_uuid=True), unique=True, default=uuid.uuid4, autoincrement=True)
name = Column(String, nullable=False, primary_key=True, unique=True)
description = Column(String, nullable=False)
def __init__(self, name: str, description: str):
self.name = name
self.description = description
@classmethod
async def find(cls, db_session: AsyncSession, name: str):
"""
:param db_session:
:param name:
:return:
"""
stmt = select(cls).where(cls.name == name)
result = await db_session.execute(stmt)
instance = result.scalars().first()
if instance is None:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail={"Not found": f"There is no record for name: {name}"},
)
else:
return instance

View File

@ -1,48 +0,0 @@
from uuid import UUID
from pydantic import BaseModel, Field
class NonsenseSchema(BaseModel):
name: str = Field(
title="",
description="",
)
description: str = Field(
title="",
description="",
)
class Config:
orm_mode = True
schema_extra = {
"example": {
"name": "Name for Some Nonsense",
"description": "Some Nonsense Description",
}
}
class NonsenseResponse(BaseModel):
id: UUID = Field(
title="Id",
description="",
)
name: str = Field(
title="",
description="",
)
description: str = Field(
title="",
description="",
)
class Config:
orm_mode = True
schema_extra = {
"example": {
"config_id": "3fa85f64-5717-4562-b3fc-2c963f66afa6",
"name": "Name for Some Nonsense",
"description": "Some Nonsense Description",
}
}

View File

@ -1,51 +0,0 @@
from __future__ import annotations
from typing import Any, Optional
from pydantic import BaseModel
class Character(BaseModel):
id: str
abbrev: str
speech_count: int
name: str
description: Any
class Chapter(BaseModel):
work_id: str
section_number: int
description: str
id: int
chapter_number: int
class Work(BaseModel):
id: str
year: int
source: str
total_paragraphs: int
title: str
long_title: str
genre_type: str
total_words: int
notes: Any
class Paragraph(BaseModel):
id: int
character_id: str
phonetic_text: str
paragraph_type: str
section_number: int
char_count: int
work_id: str
paragraph_num: int
plain_text: str
stem_text: str
chapter_number: int
word_count: int
character: Character
chapter: Chapter
work: Work

View File

@ -1,48 +0,0 @@
from uuid import UUID
from pydantic import BaseModel, Field
class StuffSchema(BaseModel):
name: str = Field(
title="",
description="",
)
description: str = Field(
title="",
description="",
)
class Config:
orm_mode = True
schema_extra = {
"example": {
"name": "Name for Some Stuff",
"description": "Some Stuff Description",
}
}
class StuffResponse(BaseModel):
id: UUID = Field(
title="Id",
description="",
)
name: str = Field(
title="",
description="",
)
description: str = Field(
title="",
description="",
)
class Config:
orm_mode = True
schema_extra = {
"example": {
"config_id": "3fa85f64-5717-4562-b3fc-2c963f66afa6",
"name": "Name for Some Stuff",
"description": "Some Stuff Description",
}
}

View File

@ -1,17 +0,0 @@
import logging
from functools import lru_cache
from rich.console import Console
from rich.logging import RichHandler
console = Console(color_system="256", width=200, style="blue")
@lru_cache()
def get_logger(module_name):
logger = logging.getLogger(module_name)
handler = RichHandler(rich_tracebacks=True, console=console, tracebacks_show_locals=True)
handler.setFormatter(logging.Formatter("[ %(threadName)s:%(funcName)s:%(lineno)d ] - %(message)s"))
logger.addHandler(handler)
logger.setLevel(logging.DEBUG)
return logger