Sqlite Changes

pebblo/app/api/api.py

@@ -1,9 +1,10 @@
-from fastapi import APIRouter
+from fastapi import APIRouter, Depends
 
-from pebblo.app.service.discovery_service import AppDiscover
+from pebblo.app.config.config import var_server_config_dict
 from pebblo.app.service.prompt_gov import PromptGov
-from pebblo.app.service.prompt_service import Prompt
-from pebblo.app.service.service import AppLoaderDoc
+from pebblo.app.utils.handler_mapper import get_handler
+
+config_details = var_server_config_dict.get()
 
 
 class App:
@@ -15,24 +16,30 @@ def __init__(self, prefix: str):
         self.router = APIRouter(prefix=prefix)
 
     @staticmethod
-    def discover(data: dict):
+    def discover(
+        data: dict, discover_obj=Depends(lambda: get_handler(handler_name="discover"))
+    ):
         # "/app/discover" API entrypoint
-        discovery_obj = AppDiscover(data=data)
-        response = discovery_obj.process_request()
+        # Execute discover object based on a storage type
+        response = discover_obj.process_request(data)
         return response
 
     @staticmethod
-    def loader_doc(data: dict):
+    def loader_doc(
+        data: dict, loader_doc_obj=Depends(lambda: get_handler(handler_name="loader"))
+    ):
         # "/loader/doc" API entrypoint
-        loader_doc_obj = AppLoaderDoc(data=data)
-        response = loader_doc_obj.process_request()
+        # Execute loader doc object based on a storage type
+        response = loader_doc_obj.process_request(data)
         return response
 
     @staticmethod
-    def prompt(data: dict):
+    def prompt(
+        data: dict, prompt_obj=Depends(lambda: get_handler(handler_name="prompt"))
+    ):
         # "/prompt" API entrypoint
-        prompt_obj = Prompt(data=data)
-        response = prompt_obj.process_request()
+        # Execute a prompt object based on a storage type
+        response = prompt_obj.process_request(data)
         return response
 
     @staticmethod

pebblo/app/config/config.py

@@ -1,11 +1,12 @@
 import pathlib
 from contextvars import ContextVar
-from typing import Tuple
+from typing import Tuple, Union
 
 import yaml
 from pydantic import BaseSettings, Field
 
 from pebblo.app.config.config_validation import validate_config, validate_input
+from pebblo.app.enums.common import DBStorageTypes, StorageTypes
 
 # Default config value
 dir_path = pathlib.Path().absolute()
@@ -45,12 +46,21 @@ class ClassifierConfig(BaseSettings):
     anonymizeSnippets: bool = Field(default=True)
 
 
+class StorageConfig(BaseSettings):
+    type: str = Field(default=StorageTypes.FILE.value)
+    db: Union[str, None] = Field(default=DBStorageTypes.SQLITE.value)
+    location: Union[str, None] = Field(default=str(dir_path))
+    name: Union[str, None] = Field(default=str("pebblo"))
+    # This is default value for current version(0.1.18), it needs to be changed in next version to db.
+
+
 # ConfigFile BaseModel
 class Config(BaseSettings):
     daemon: PortConfig
     reports: ReportConfig
     logging: LoggingConfig
     classifier: ClassifierConfig
+    storage: StorageConfig
 
 
 var_server_config: ContextVar[Config] = ContextVar("server_config", default=None)
@@ -67,6 +77,8 @@ def load_config(path: str) -> Tuple[dict, Config]:
             ),
             logging=LoggingConfig(),
             classifier=ClassifierConfig(anonymizeSnippets=False),
+            storage=StorageConfig(type="file", db=None),
+            # for now, a default storage type is FILE, but in the next release DB will be the default storage type.
         )
         if not path:
             # Setting Default config details
@@ -95,4 +107,4 @@ def load_config(path: str) -> Tuple[dict, Config]:
 
     except Exception as err:
         print(f"Error while loading config details, err: {err}")
-        return {}
+        return {}, {}

pebblo/app/config/config.yaml

@@ -9,3 +9,5 @@ reports:
   cacheDir: ~/.pebblo
 classifier:
   anonymizeSnippets: False
+storage:
+  type: file

pebblo/app/config/config_validation.py

@@ -3,6 +3,8 @@
 import sys
 from abc import ABC, abstractmethod
 
+from pebblo.app.enums.common import DBStorageTypes, StorageTypes
+
 
 class ConfigValidator(ABC):
     def __init__(self, config):
@@ -42,6 +44,45 @@ def validate(self):
             )
 
 
+class StorageConfig(ConfigValidator):
+    def validate(self):
+        storage_type = self.config.get("type")
+        valid_storage_types = [storage_type.value for storage_type in StorageTypes]
+        if storage_type not in valid_storage_types:
+            self.errors.append(
+                f"Error: Unsupported storage type '{storage_type}' specified in the configuration."
+                f"Valid values are {valid_storage_types}"
+            )
+
+        # Set deprecated warning message for file storage type
+        if storage_type == StorageTypes.FILE.value:
+            deprecate_error = f"DeprecationWarning: '{storage_type}' Storage Type will be deprecated starting from Pebblo version 0.0.19, use '{StorageTypes.DATABASE.value}' instead"
+            print(deprecate_error)
+
+        if storage_type == StorageTypes.DATABASE.value:
+            db_type = self.config.get("db")
+            default_location = self.config.get("location")
+            db_name = self.config.get("name")
+
+            valid_db_types = [storage_type.value for storage_type in DBStorageTypes]
+            if db_type not in valid_db_types:
+                self.errors.append(
+                    f"Error: Unsupported db type '{db_type}' specified in the configuration."
+                    f"Valid values are {valid_db_types}"
+                )
+
+            # db_name should be in string
+            if not isinstance(db_name, str):
+                self.errors.append(
+                    f"Error: Unsupported db name '{db_name} specified in the configuration"
+                    f"String values are allowed only"
+                )
+
+            # Check if the output directory exists, create if it doesn't
+            if not os.path.exists(expand_path(str(default_location))):
+                os.makedirs(expand_path(str(default_location)), exist_ok=True)
+
+
 def expand_path(file_path: str) -> str:
     # Expand user (~) and environment variables
     expanded_path = os.path.expanduser(file_path)
@@ -102,6 +143,7 @@ def validate_config(config_dict):
         "logging": LoggingConfig,
         "reports": ReportsConfig,
         "classifier": ClassifierConfig,
+        "storage": StorageConfig,
     }
 
     validation_errors = []

pebblo/app/config/service.py

@@ -13,10 +13,10 @@
 from pebblo.app.exceptions.exception_handler import exception_handlers
 from pebblo.app.routers.local_ui_routers import local_ui_router_instance
 from pebblo.app.routers.redirection_router import redirect_router_instance
-from pebblo.log import get_logger, get_uvicorn_logconfig
 
 with redirect_stdout(StringIO()), redirect_stderr(StringIO()):
     from pebblo.app.routers.routers import router_instance
+from pebblo.log import get_logger, get_uvicorn_logconfig
 
 logger = get_logger(__name__)
 

pebblo/app/enums/common.py

@@ -0,0 +1,11 @@
+from enum import Enum
+
+
+class StorageTypes(Enum):
+    FILE = "file"
+    DATABASE = "db"
+
+
+class DBStorageTypes(Enum):
+    SQLITE = "sqlite"
+    MONGODB = "mongodb"

pebblo/app/enums/enums.py

@@ -30,6 +30,11 @@ class CacheDir(Enum):
         f"http://{config_details.get('daemon', {}).get('host')}:"
         f"{config_details.get('daemon', {}).get('port')}"
     )
+    DB_NAME = config_details.get("storage", {}).get("name")
+    DB_LOCATION = config_details.get("storage", {}).get("location")
+
+    SQLITE_ENGINE = "sqlite:///{}/pebblo.db"
+    # SQLITE_ENGINE = f"sqlite:///{DB_LOCATION}/{DB_NAME}.db"
 
 
 class ReportConstants(Enum):