Merge pull request #28 from Wytamma/no-sql

v1.5.0

Author: Wytamma

backend/beastiary/api/core.py

@@ -2,10 +2,8 @@
 from pathlib import Path
 from typing import Optional, Tuple, Union, List
 from beastiary import crud, schemas
-from beastiary.models.trace import Trace
+from beastiary.db.database import Database
 from pydantic.utils import is_valid_field
-from sqlalchemy.orm.session import Session
-from beastiary.schemas.sample import SampleCreate
 from beastiary.log import logger
 import os, math, errno
 
@@ -39,45 +37,41 @@ def is_valid_log_file(headers_line: str, delimiter: Optional[str] = None) -> boo
     return False
 
 
-def add_trace(db: Session, trace_in: schemas.TraceCreate) -> Trace:
+def add_trace(db: dict, trace_in: schemas.TraceCreate) -> dict:
     if not trace_in.path.is_file():
         raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), trace_in.path)
     last_byte, headers_line = get_headers(trace_in.path, delimiter=trace_in.delimiter)
     if not is_valid_log_file(headers_line, delimiter=trace_in.delimiter):
         raise ValueError(f"Invalid log file: {trace_in.path}")
     logger.debug(f"Creating trace: {trace_in}")
     trace = crud.trace.create(
-        db=db, obj_in=trace_in, headers_line=headers_line, last_byte=last_byte
+        db, obj_in=trace_in, last_byte=last_byte, headers_line=headers_line
     )
     logger.debug(f"Created trace: {trace}")
     return trace
 
 
-def read_lines(trace: Trace) -> Tuple[int, list]:
+def read_lines(trace: schemas.Trace) -> Tuple[int, list]:
     logger.debug(f"reading lines from: {trace}")
-    if not trace.path:
+    if not trace["path"]:
         raise ValueError("Path must be set.")
-    with open(trace.path, "r") as f:
-        f.seek(trace.last_byte, 0)
+    with open(trace["path"], "r") as f:
+        f.seek(trace["last_byte"], 0)
         lines = f.readlines()
         if lines:
             last_byte = f.tell()
         else:
-            last_byte = trace.last_byte
+            last_byte = trace["last_byte"]
         logger.debug(f"last_byte = {last_byte}")
         logger.debug(f"lines found = {len(lines)}")
         return last_byte, lines
 
 
 def lines_to_SampleCreate(
-    headers: list, lines: list, delimiter: Optional[str] = None
-) -> List[SampleCreate]:
+    headers: list, lines: list, trace_id: int, delimiter: Optional[str] = None
+) -> List[dict]:
     samples = []
     for line in lines:
-        if not line:
-            # blank lines are bad...
-            # you'll have to have a smart way to handel this with the byte offset
-            raise ValueError("Poorly formated line.")
         data = {}
         line = line.strip()  # strip \n
         for header, value in zip(headers, line.split(delimiter)):
@@ -90,18 +84,20 @@ def lines_to_SampleCreate(
         sample_in = {}
         sample_in["data"] = data
         sample_in["state"] = data["state"]
-        samples.append(schemas.sample.SampleCreate(**sample_in))
+        sample_in["trace_id"] = trace_id
+        samples.append(sample_in)
     return samples
 
 
-def check_for_new_samples(
-    db: Session, trace: Trace, delimiter: Optional[str] = None
-) -> None:
+def check_for_new_samples(db: Database, trace: schemas.Trace) -> None:
     last_byte, lines = read_lines(trace)
     in_samples = lines_to_SampleCreate(
-        trace.headers_line.split(delimiter), lines, delimiter=trace.delimiter
+        trace["headers_line"].split(trace["delimiter"]),
+        lines,
+        trace_id=trace["id"],
+        delimiter=trace["delimiter"],
     )
     if in_samples:
-        crud.sample.create_multi_with_trace(db, objs_in=in_samples, trace_id=trace.id)
+        crud.sample.create_multi(db, objs_in=in_samples)
     # update the trace byte
     crud.trace.update(db, db_obj=trace, obj_in={"last_byte": last_byte})

backend/beastiary/api/endpoints/samples.py

@@ -1,46 +1,41 @@
-from typing import Any, List, Tuple
-import math
-from beastiary.models.trace import Trace
-from fastapi import APIRouter, Depends, HTTPException, Path
-from sqlalchemy.orm import Session
+from typing import Any, List
+from fastapi import APIRouter, Depends, HTTPException, Path, Request
 
-from beastiary import crud, schemas
-from beastiary.api import deps
+from beastiary import crud
 from beastiary.api.core import check_for_new_samples
 from beastiary.log import logger
 
 router = APIRouter()
 
 
-@router.get("/", response_model=List[schemas.Sample])
+@router.get("/")
 def get_samples(
+    request: Request,
     trace_id: int,
     skip: int = 0,
     limit: int = 100,
-    db: Session = Depends(deps.get_db),
 ) -> Any:
     """
     Retrieve samples.
     """
-    trace = crud.trace.get(db, trace_id)
+    trace = crud.trace.get(request.app.db, trace_id)
     if not trace:
         raise HTTPException(404, detail="Trace not found!")
 
     samples = crud.sample.get_multi_by_trace(
-        db, trace_id=trace.id, skip=skip, limit=limit
+        request.app.db, trace_id=trace["id"], skip=skip, limit=limit
     )
-
     logger.debug(f"{limit} samples requested - {len(samples)} found")
     if len(samples) < limit:
-
-        logger.debug(f"Checking for new samples in {trace.path}")
+        logger.debug(f"Checking for new samples in {trace['path']}")
         try:
-            check_for_new_samples(db, trace=trace, delimiter=trace.delimiter)
+            check_for_new_samples(request.app.db, trace=trace)
         except Exception as e:
-            raise HTTPException(500, detail=f"Could read samples in {trace.path}")
+            logger.error(str(e))
+            raise HTTPException(500, detail=f"Could read samples in {trace['path']}")
         # get samples
         samples = crud.sample.get_multi_by_trace(
-            db, trace_id=trace.id, skip=skip, limit=limit
+            request.app.db, trace_id=trace["id"], skip=skip, limit=limit
         )
     logger.debug(f"Returning {len(samples)} samples")
     return samples

backend/beastiary/api/endpoints/traces.py

@@ -1,51 +1,47 @@
 from beastiary.api.core import add_trace
-from beastiary.models.trace import Trace
-from beastiary.schemas import sample
 from typing import Any, List
-from fastapi import APIRouter, Depends, HTTPException
-from sqlalchemy.orm import Session
+from fastapi import APIRouter, Depends, HTTPException, Request
 
 from beastiary import crud, schemas
-from beastiary.api import deps
 
 router = APIRouter()
 
 
 @router.get("/", response_model=List[schemas.Trace])
 def get_traces(
-    db: Session = Depends(deps.get_db),
+    request: Request,
     skip: int = 0,
     limit: int = 100,
-) -> List[Trace]:
+) -> List[schemas.Trace]:
     """
     Retrieve traces.
     """
-    traces = crud.trace.get_multi(db=db, skip=skip, limit=limit)
+    traces = crud.trace.get_multi(db=request.app.db, skip=skip, limit=limit)
     return traces
 
 
 @router.get("/{trace_id}", response_model=schemas.Trace)
-def get_trace(trace_id: int, db: Session = Depends(deps.get_db)) -> Trace:
+def get_trace(request: Request, trace_id: int) -> dict:
     """
     Retrieve traces.
     """
-    trace = crud.trace.get(db=db, id=trace_id)
+    trace = crud.trace.get(db=request.app.db, id=trace_id)
     if not trace:
         raise HTTPException(404, "Trace not found!")
     return trace
 
 
 @router.post("/", response_model=schemas.Trace)
 def create_trace(
+    request: Request,
     *,
-    db: Session = Depends(deps.get_db),
     trace_in: schemas.TraceCreate,
-) -> Trace:
+) -> dict:
     """
     Create new trace.
     """
     try:
-        trace = add_trace(db, trace_in)
+        trace = add_trace(request.app.db, trace_in)
     except FileNotFoundError as e:
         raise HTTPException(404, detail="Could not find log file!")
     except ValueError as e:

backend/beastiary/cli.py

@@ -7,20 +7,14 @@
 
 from pathlib import Path
 from typing import List, Optional
-from fastapi.params import Security
 
-from beastiary.api.core import add_trace
+from beastiary.api.core import add_trace, check_for_new_samples
 from beastiary.api import api
-from beastiary.db.init_db import init_db
-from beastiary.db.session import SessionLocal
+from beastiary.db import Database
 from beastiary import schemas
 
-
 app = typer.Typer()
 
-db = SessionLocal()
-init_db(db)
-
 
 @app.command()
 def main(
@@ -43,6 +37,10 @@ def main(
     """
     Realtime and remote trace inspection with BEASTIARY.
     """
+    db = Database()
+    db.create_table("Trace")
+    db.create_table("Sample")
+    setattr(api, "db", db)
     if version:
         typer.echo(f"Beastiary {pkg_resources.get_distribution('beastiary').version}")
         return typer.Exit()
@@ -53,9 +51,10 @@ def main(
         for path in log_files:
             try:
                 trace = add_trace(
-                    db, schemas.TraceCreate(path=str(path), delimiter=delimiter)
+                    api.db, schemas.TraceCreate(path=str(path), delimiter=delimiter)
                 )
-                typer.echo(f"✅ - {trace.path}")
+                check_for_new_samples(api.db, trace=trace)
+                typer.echo(f"✅ - {trace['path']}")
             except ValueError:
                 typer.echo(f"❌ - {path}")
         typer.echo("")

backend/beastiary/crud/base.py

@@ -1,67 +1,43 @@
 from typing import Any, Dict, Generic, List, Optional, Type, TypeVar, Union
+from beastiary.db.database import Database
 
 from fastapi.encoders import jsonable_encoder
 from pydantic import BaseModel
-from sqlalchemy.orm import Session
 
-from beastiary.db.base_class import Base
-
-ModelType = TypeVar("ModelType", bound=Base)
+InDBSchemaType = TypeVar("InDBSchemaType", bound=BaseModel)
 CreateSchemaType = TypeVar("CreateSchemaType", bound=BaseModel)
 UpdateSchemaType = TypeVar("UpdateSchemaType", bound=BaseModel)
 
 
-class CRUDBase(Generic[ModelType, CreateSchemaType, UpdateSchemaType]):
-    def __init__(self, model: Type[ModelType]):
+class CRUDBase(Generic[InDBSchemaType, CreateSchemaType, UpdateSchemaType]):
+    def __init__(self, model: Type[InDBSchemaType]):
         """
         CRUD object with default methods to Create, Read, Update, Delete (CRUD).
         **Parameters**
-        * `model`: A SQLAlchemy model class
+        * `model`: A Pydantic model (schema) class for in database representation
         * `schema`: A Pydantic model (schema) class
         """
         self.model = model
 
-    def get(self, db: Session, id: Any) -> Optional[ModelType]:
-        return db.query(self.model).filter(self.model.id == id).first()
+    def get(self, db: Database, id: Any) -> Optional[InDBSchemaType]:
+        return db.query(self.model.__name__, id=id)
 
     def get_multi(
-        self, db: Session, *, skip: int = 0, limit: int = 100
-    ) -> List[ModelType]:
-        return db.query(self.model).offset(skip).limit(limit).all()
+        self, db: Database, *, skip: int = 0, limit: int = 100
+    ) -> List[InDBSchemaType]:
+        return db.query(self.model.__name__, skip=skip, limit=limit)
 
-    def create(self, db: Session, *, obj_in: CreateSchemaType) -> ModelType:
-        obj_in_data = jsonable_encoder(obj_in)
-        db_obj = self.model(**obj_in_data)  # type: ignore
-        db.add(db_obj)
-        db.commit()
-        db.refresh(db_obj)
+    def create(self, db: Database, *, obj_in: CreateSchemaType) -> InDBSchemaType:
+        db_obj = jsonable_encoder(obj_in)
+        db.add(self.model.__name__, db_obj)
         return db_obj
 
     def update(
         self,
-        db: Session,
+        db: Database,
         *,
-        db_obj: ModelType,
+        db_obj: dict,
         obj_in: Union[UpdateSchemaType, Dict[str, Any]],
-    ) -> ModelType:
-        db.refresh(db_obj)
-        obj_data = jsonable_encoder(db_obj)
-        if isinstance(obj_in, dict):
-            update_data = obj_in
-        else:
-            update_data = obj_in.dict(exclude_unset=True)
-        for field in obj_data:
-            if field in update_data:
-                setattr(db_obj, field, update_data[field])
-        db.add(db_obj)
-        db.commit()
-        db.refresh(db_obj)
+    ) -> InDBSchemaType:
+        db_obj.update(obj_in)
         return db_obj
-
-    def remove(self, db: Session, *, id: int) -> ModelType:
-        obj = db.query(self.model).get(id)
-        if not obj:
-            raise ValueError(f"{obj} not found!")
-        db.delete(obj)
-        db.commit()
-        return obj

backend/beastiary/crud/crud_sample.py

@@ -1,46 +1,21 @@
 from typing import List
+from beastiary.db.database import Database
 
 from fastapi.encoders import jsonable_encoder
-from sqlalchemy.orm import Session
 
 from beastiary.crud.base import CRUDBase
-from beastiary.models.sample import Sample
-from beastiary.schemas.sample import SampleCreate, SampleUpdate
+from beastiary.schemas.sample import SampleCreate, SampleUpdate, Sample
 
 
 class CRUDSample(CRUDBase[Sample, SampleCreate, SampleUpdate]):
-    def create_with_trace(
-        self, db: Session, *, obj_in: SampleCreate, trace_id: int
-    ) -> Sample:
-        obj_in_data = jsonable_encoder(obj_in)
-        db_obj = self.model(**obj_in_data, trace_id=trace_id)
-        db.add(db_obj)
-        db.commit()
-        db.refresh(db_obj)
-        return db_obj
-
-    def create_multi_with_trace(
-        self, db: Session, *, objs_in: List[SampleCreate], trace_id: int
-    ) -> List[Sample]:
-        objs_in_data = [jsonable_encoder(obj_in) for obj_in in objs_in]
-        for obj_in in objs_in_data:
-            obj_in.update({"trace_id": trace_id})
-        db_objs = [self.model(**obj_in_data) for obj_in_data in objs_in_data]
-        db.add_all(db_objs)
-        db.commit()
-        # db.refresh(db_objs)
-        return db_objs
+    def create_multi(self, db: Database, *, objs_in: List[dict]) -> List[Sample]:
+        db.add_all(self.model.__name__, objs_in)
+        return objs_in
 
     def get_multi_by_trace(
-        self, db: Session, *, trace_id: int, skip: int = 0, limit: int = 100
+        self, db: Database, *, trace_id: int, skip: int = 0, limit: int = 100
     ) -> List[Sample]:
-        return (
-            db.query(self.model)
-            .filter(Sample.trace_id == trace_id)
-            .offset(skip)
-            .limit(limit)
-            .all()
-        )
+        return db.query(self.model.__name__, skip=skip, limit=limit, trace_id=trace_id)
 
 
 sample = CRUDSample(Sample)