Refactored and add slack_loader

.gitignore

@@ -3,7 +3,7 @@
 .vscode/
 .vscode/launch.json
 backup
-chroma_boardgames/
+vector_store/
 env/
 .env
 __pycache__

README.md

@@ -11,8 +11,7 @@ Clone this repository and create a clean python v3.10 virtual environment and ac
 The following is a high-level list of components used to run this local RAG:
 
 - langchain
-- streamlit
-- streamlit-chat
+- ollama
 - pypdf
 - chromadb
 - fastembed
@@ -37,7 +36,7 @@ Create a `.env` file in the root directory and add the following environment var
 
 ```.env
 
-CHROMA_PATH=chroma_boardgames
+CHROMA_DB_PATH_PDF=chroma_boardgames
 DATA_PATH_BG=data_boardgames
 ```
 
@@ -53,7 +52,7 @@ If you need to clear the database for any reason, run:
 
 The above command will remove the chroma database. If you need to recreate it, simply rerun `populate_database.py`
 
-## Running the RAG
+## Running the RAG from the commandline:
 
 The instruction manuals for both Monopoly and Ticket To Ride have been loaded into the Chroma DB. Ask the RAG questions about these two board games and see how well it does answering your questions. The RAG can be invoked using the following command with the sample question:
 
@@ -68,6 +67,14 @@ Here are some additional questions you can try:
 
 You can also browse the instruction manuals that are in the `./data_boardgames` folder to come up with your own questions.
 
+## Running the FASTAPI server to expose the RAG via API
+
+Start the FASTPI server to expose api's that can be called from the ai_rag_ui or curl.
+
+```
+python query_data.py  How do I build a hotel in monopoly?
+```
+
 ## Running the test cases
 
 ```

api.py → ai_local_rag/api.py

@@ -1,11 +1,13 @@
 import logging
-from query_data import query_rag
-from fastapi import FastAPI, Request, HTTPException, status
+
+from fastapi import FastAPI, HTTPException, Request, status
 from fastapi.exceptions import RequestValidationError
 from fastapi.responses import JSONResponse
 from pydantic import BaseModel
-from models.rag_query_model import QueryInput, QueryOutput
-from utils.async_utils import async_retry
+
+from ai_local_rag.models.rag_query_model import QueryInput, QueryOutput
+from ai_local_rag.query_data import query_rag
+from ai_local_rag.utils.async_utils import async_retry
 
 
 class Message(BaseModel):
@@ -23,6 +25,10 @@ class Message(BaseModel):
 channel_list = ["general", "dev", "marketing"]
 message_map = {}
 
+logging.basicConfig()
+logger = logging.getLogger("api")
+logger.setLevel(logging.DEBUG)
+
 
 @app.exception_handler(RequestValidationError)
 async def validation_exception_handler(request: Request, exc: RequestValidationError):
@@ -60,20 +66,21 @@ def post_message(message: Message):
 
 @app.post("/rag-query")
 async def query_rag_api(query: QueryInput):
-    print(f"api.py - API Request Data: {query}")
+    logger.info(f"api.py - API Request Data: {query}")
     query_response = query_rag({"input": query})
-    print(query_response)
+    logger.debug(f"Query Response: - {query_response}")
 
     # query: str
     # response: str
     # sources: list[str]
     query_response2 = {
         "query": query, "response": query_response["response"], "sources": query_response["sources"]}
-    print(f"Query Response2:  {query_response2}")
+    logger.info(f"Query Response2:  {query_response2}")
 
     # query_response["intermediate_steps"] = [
     #    str(s) for s in query_response["intermediate_steps"]
     # ]
+    logger.debug(f"Query Response: - {query_response2}")
 
     return query_response2
 
@@ -83,15 +90,15 @@ async def query_rag_api2(
     query: QueryInput,
 ) -> QueryOutput:
     query_response = query_rag({"input": query})
-    print(query_response)
+    logger.debug(f"Query Response: {query_response}")
 
     # query: str
     # response: str
     # sources: list[str]
     query_text = query["query"]
     query_response2 = {
         "query": query_text, "response": query_response["response"], "sources": query_response["sources"]}
-    print(f"Query Response2:  {query_response2}")
+    logger.debug(f"Query Response2:  {query_response2}")
 
     # query_response["intermediate_steps"] = [
     #    str(s) for s in query_response["intermediate_steps"]

ai_local_rag/chroma/chroma_client.py

@@ -0,0 +1,36 @@
+from chromadb.config import Settings
+import chromadb
+import logging
+import os
+
+import dotenv
+from langchain.vectorstores import Chroma
+
+# from ai_local_rag.utils.get_embedding_function import get_embedding_function
+
+
+dotenv.load_dotenv()
+
+
+logging.basicConfig()
+logger = logging.getLogger("chroma_client")
+logger.setLevel(logging.DEBUG)
+
+
+chroma_path = os.getenv("CHROMA_PATH_SLACK")
+collection_name = os.getenv("CHROMA_SLACK_COLLECTION")
+
+
+client = chromadb.Client(Settings(
+    persist_directory=chroma_path
+))
+
+logger.info(f"Number of collections:  {client.count_collections()}")
+
+logger.info(client.list_collections())
+"""
+collection = client.get_collection(name=collection_name)
+result = collection.query(n_results=5)
+logging.info(result)
+logging.info(len(client.list_collections()))
+"""

populate_database.py → ai_local_rag/chroma/populate_database.py

@@ -1,16 +1,16 @@
 import argparse
 import os
-from dotenv import load_dotenv
-
-
 import shutil
+
+from dotenv import load_dotenv
+from langchain.schema.document import Document
 # from langchain.document_loaders.pdf import PyPDFDirectoryLoader
 from langchain_community.document_loaders import PyPDFDirectoryLoader
-from langchain_text_splitters import RecursiveCharacterTextSplitter
-from langchain.schema.document import Document
-from get_embedding_function import get_embedding_function
 # from langchain.vectorstores.chroma import Chroma
 from langchain_community.vectorstores import Chroma
+from langchain_text_splitters import RecursiveCharacterTextSplitter
+
+from ai_local_rag.utils.get_embedding_function import get_embedding_function_for_pdf
 
 # Load Config Settings
 load_dotenv()  # take environment variables from .env.
@@ -56,10 +56,10 @@ def _split_documents(documents: list[Document]):
 
 
 def _add_to_chroma(chunks: list[Document]):
-    chroma_path = os.getenv("CHROMA_PATH")
+    chroma_db_path_pdf = os.getenv("CHROMA_DB_PATH_PDF")
     # Load the existing database.
     db = Chroma(
-        persist_directory=chroma_path, embedding_function=get_embedding_function()
+        persist_directory=chroma_db_path_pdf, embedding_function=get_embedding_function_for_pdf()
     )
 
     # Calculate Page IDs.
@@ -119,10 +119,10 @@ def clear_database():
     Remove the database so that we can rebuild it
     """
     load_dotenv()  # take environment variables from .env.
-    chroma_path = os.getenv("CHROMA_PATH")
+    chroma_db_path = os.getenv("CHROMA_DB_PATH_PDF")
 
-    if os.path.exists(chroma_path):
-        shutil.rmtree(chroma_path)
+    if os.path.exists(chroma_db_path):
+        shutil.rmtree(chroma_db_path)
 
 
 if __name__ == "__main__":

ai_local_rag/chroma/reset_database.py

@@ -0,0 +1,10 @@
+import os
+from dotenv import load_dotenv
+import ai_local_rag.chroma.populate_database as populate_database
+
+# Load Config Settings
+load_dotenv()  # take environment variables from .env.
+chroma_db_path_pdf = os.getenv("CHROMA_DB_PATH_PDF")
+
+populate_database.clear_database()
+print(f"Removed all content from database {chroma_db_path_pdf}")