diff --git a/README.md b/README.md index ca2f397e..86a3e839 100644 --- a/README.md +++ b/README.md @@ -144,7 +144,7 @@ Examples of these classes are: Code example for creating an instance of 'SymbolCodeEmbedding': ```python import numpy as np -from automata.core.symbol_embedding.base import SymbolCodeEmbedding +from automata.core.base.symbol_embedding import SymbolCodeEmbedding from automata.core.symbol.parser import parse_symbol symbol_str = 'scip-python python automata 75482692a6fe30c72db516201a6f47d9fb4af065 `automata.core.agent.agent_enums`/ActionIndicator#' @@ -157,7 +157,7 @@ embedding = SymbolCodeEmbedding(symbol=symbol, source_code=source_code, vector=v Code example for creating an instance of 'SymbolDocEmbedding': ```python -from automata.core.symbol.base import SymbolDocEmbedding +from automata.core.base.symbol_embedding import SymbolDocEmbedding from automata.core.symbol.parser import parse_symbol import numpy as np diff --git a/automata/cli/scripts/run_code_embedding.py b/automata/cli/scripts/run_code_embedding.py index 84fc7ab2..35477099 100644 --- a/automata/cli/scripts/run_code_embedding.py +++ b/automata/cli/scripts/run_code_embedding.py @@ -4,13 +4,13 @@ from tqdm import tqdm from automata.config.base import ConfigCategory -from automata.core.base.database.vector import JSONEmbeddingVectorDatabase +from automata.core.base.symbol_embedding import JSONSymbolEmbeddingVectorDatabase from automata.core.coding.py.module_loader import py_module_loader from automata.core.llm.providers.openai import OpenAIEmbeddingProvider from automata.core.memory_store.symbol_code_embedding import SymbolCodeEmbeddingHandler from automata.core.symbol.graph import SymbolGraph from automata.core.symbol.symbol_utils import get_rankable_symbols -from automata.core.symbol_embedding.embedding_builders import SymbolCodeEmbeddingBuilder +from automata.core.symbol_embedding.builders import SymbolCodeEmbeddingBuilder from automata.core.utils import get_config_fpath logger = logging.getLogger(__name__) @@ -37,7 +37,7 @@ def main(*args, **kwargs) -> str: all_defined_symbols = symbol_graph.get_all_available_symbols() filtered_symbols = sorted(get_rankable_symbols(all_defined_symbols), key=lambda x: x.dotpath) - embedding_db = JSONEmbeddingVectorDatabase(embedding_path) + embedding_db = JSONSymbolEmbeddingVectorDatabase(embedding_path) embedding_provider = OpenAIEmbeddingProvider() embedding_builder = SymbolCodeEmbeddingBuilder(embedding_provider) embedding_handler = SymbolCodeEmbeddingHandler(embedding_db, embedding_builder) diff --git a/automata/cli/scripts/run_doc_embedding_l2.py b/automata/cli/scripts/run_doc_embedding_l2.py index 56a0a876..62791b3c 100644 --- a/automata/cli/scripts/run_doc_embedding_l2.py +++ b/automata/cli/scripts/run_doc_embedding_l2.py @@ -4,7 +4,8 @@ from tqdm import tqdm from automata.config.base import ConfigCategory -from automata.core.base.database.vector import JSONEmbeddingVectorDatabase +from automata.core.base.symbol import SymbolDescriptor +from automata.core.base.symbol_embedding import JSONSymbolEmbeddingVectorDatabase from automata.core.coding.py.module_loader import py_module_loader from automata.core.context.py.retriever import ( PyContextRetriever, @@ -16,12 +17,11 @@ ) from automata.core.memory_store.symbol_code_embedding import SymbolCodeEmbeddingHandler from automata.core.memory_store.symbol_doc_embedding import SymbolDocEmbeddingHandler -from automata.core.symbol.base import SymbolDescriptor from automata.core.symbol.graph import SymbolGraph from automata.core.symbol.search.rank import SymbolRankConfig from automata.core.symbol.search.symbol_search import SymbolSearch from automata.core.symbol.symbol_utils import get_rankable_symbols -from automata.core.symbol_embedding.embedding_builders import ( +from automata.core.symbol_embedding.builders import ( SymbolCodeEmbeddingBuilder, SymbolDocEmbeddingBuilder, ) @@ -46,7 +46,7 @@ def main(*args, **kwargs) -> str: code_embedding_fpath = os.path.join( get_config_fpath(), ConfigCategory.SYMBOL.value, "symbol_code_embedding.json" ) - code_embedding_db = JSONEmbeddingVectorDatabase(code_embedding_fpath) + code_embedding_db = JSONSymbolEmbeddingVectorDatabase(code_embedding_fpath) embedding_provider = OpenAIEmbeddingProvider() embedding_builder = SymbolCodeEmbeddingBuilder(embedding_provider) code_embedding_handler = SymbolCodeEmbeddingHandler(code_embedding_db, embedding_builder) @@ -57,7 +57,7 @@ def main(*args, **kwargs) -> str: ConfigCategory.SYMBOL.value, kwargs.get("symbol_doc_embedding_l2_fpath", "symbol_doc_embedding_l2.json"), ) - embedding_db_l2 = JSONEmbeddingVectorDatabase(embedding_path_l2) + embedding_db_l2 = JSONSymbolEmbeddingVectorDatabase(embedding_path_l2) symbol_graph = SymbolGraph(scip_path) diff --git a/automata/cli/scripts/run_doc_embedding_l3.py b/automata/cli/scripts/run_doc_embedding_l3.py index 31439031..34e124df 100644 --- a/automata/cli/scripts/run_doc_embedding_l3.py +++ b/automata/cli/scripts/run_doc_embedding_l3.py @@ -4,7 +4,8 @@ from tqdm import tqdm from automata.config.base import ConfigCategory -from automata.core.base.database.vector import JSONEmbeddingVectorDatabase +from automata.core.base.symbol import SymbolDescriptor +from automata.core.base.symbol_embedding import JSONSymbolEmbeddingVectorDatabase from automata.core.coding.py.module_loader import py_module_loader from automata.core.context.py.retriever import ( PyContextRetriever, @@ -16,12 +17,11 @@ ) from automata.core.memory_store.symbol_code_embedding import SymbolCodeEmbeddingHandler from automata.core.memory_store.symbol_doc_embedding import SymbolDocEmbeddingHandler -from automata.core.symbol.base import SymbolDescriptor from automata.core.symbol.graph import SymbolGraph from automata.core.symbol.search.rank import SymbolRankConfig from automata.core.symbol.search.symbol_search import SymbolSearch from automata.core.symbol.symbol_utils import get_rankable_symbols -from automata.core.symbol_embedding.embedding_builders import ( +from automata.core.symbol_embedding.builders import ( SymbolCodeEmbeddingBuilder, SymbolDocEmbeddingBuilder, ) @@ -45,7 +45,7 @@ def main(*args, **kwargs) -> str: code_embedding_fpath = os.path.join( get_config_fpath(), ConfigCategory.SYMBOL.value, "symbol_code_embedding.json" ) - code_embedding_db = JSONEmbeddingVectorDatabase(code_embedding_fpath) + code_embedding_db = JSONSymbolEmbeddingVectorDatabase(code_embedding_fpath) embedding_provider = OpenAIEmbeddingProvider() embedding_builder = SymbolCodeEmbeddingBuilder(embedding_provider) code_embedding_handler = SymbolCodeEmbeddingHandler(code_embedding_db, embedding_builder) @@ -56,7 +56,7 @@ def main(*args, **kwargs) -> str: ConfigCategory.SYMBOL.value, kwargs.get("symbol_doc_embedding_l2_fpath", "symbol_doc_embedding_l2.json"), ) - embedding_db_l2 = JSONEmbeddingVectorDatabase(embedding_path_l2) + embedding_db_l2 = JSONSymbolEmbeddingVectorDatabase(embedding_path_l2) embedding_path_l3 = os.path.join( get_config_fpath(), @@ -66,7 +66,7 @@ def main(*args, **kwargs) -> str: symbol_graph = SymbolGraph(scip_path) - embedding_db_l3 = JSONEmbeddingVectorDatabase(embedding_path_l3) + embedding_db_l3 = JSONSymbolEmbeddingVectorDatabase(embedding_path_l3) symbol_code_similarity = SymbolSimilarityCalculator(code_embedding_handler, embedding_provider) diff --git a/automata/cli/scripts/run_doc_post_process.py b/automata/cli/scripts/run_doc_post_process.py index 5c57f6fc..911363a4 100644 --- a/automata/cli/scripts/run_doc_post_process.py +++ b/automata/cli/scripts/run_doc_post_process.py @@ -2,7 +2,7 @@ import os from automata.config.base import ConfigCategory -from automata.core.base.database.vector import JSONEmbeddingVectorDatabase +from automata.core.base.symbol_embedding import JSONSymbolEmbeddingVectorDatabase from automata.core.coding.py.writer import PyDocWriter from automata.core.utils import get_config_fpath, get_root_py_fpath @@ -19,7 +19,7 @@ def main(*args, **kwargs) -> str: get_config_fpath(), ConfigCategory.SYMBOL.value, "symbol_doc_embedding_l2.json" ) - embedding_db = JSONEmbeddingVectorDatabase(embedding_path) + embedding_db = JSONSymbolEmbeddingVectorDatabase(embedding_path) symbols = [embedding.symbol for embedding in embedding_db.get_all_entries()] diff --git a/automata/config/symbol/index.scip b/automata/config/symbol/index.scip index b5f48812..b9fb402f 100644 Binary files a/automata/config/symbol/index.scip and b/automata/config/symbol/index.scip differ diff --git a/automata/config/symbol/symbol_code_embedding.json b/automata/config/symbol/symbol_code_embedding.json index fd4b2f79..3c13109e 100644 --- a/automata/config/symbol/symbol_code_embedding.json +++ b/automata/config/symbol/symbol_code_embedding.json @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:462d7cb1299826b4d0c79b4583addc344ef72b374e3f3156b6039cb0a84e6ba6 -size 14393187 +oid sha256:d4a8f3c6915727bc6afd35176ddc9647f961e3f2080e7c57a05fd72ad69d2a89 +size 14238278 diff --git a/automata/config/symbol/symbol_doc_embedding_l2.json b/automata/config/symbol/symbol_doc_embedding_l2.json index 4f9cecf1..a8e76b3d 100644 --- a/automata/config/symbol/symbol_doc_embedding_l2.json +++ b/automata/config/symbol/symbol_doc_embedding_l2.json @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:102f68cb8b8b5dc8ca810e86cb83e14dbf62abcd70c6da24b569be7af05b614f -size 5157496 +oid sha256:09161c58c3b717ea2fed83396cfdb9785d69a35f53b7e74fb0d92a314fb38f61 +size 4969252 diff --git a/automata/config/symbol/symbol_doc_embedding_l3.json b/automata/config/symbol/symbol_doc_embedding_l3.json index 7f535f68..47c8eab6 100644 --- a/automata/config/symbol/symbol_doc_embedding_l3.json +++ b/automata/config/symbol/symbol_doc_embedding_l3.json @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:0ab289ef890683e90465a0c38a7f63561f2ae5e0814e06ab8dad731f56870ff1 -size 5778177 +oid sha256:ed0ee7a0660103b9755f1de09d9290359a1921fa3b340217b17390f94fafdaf1 +size 5653605 diff --git a/automata/core/agent/providers.py b/automata/core/agent/providers.py index 15a9ea7c..0633d52f 100644 --- a/automata/core/agent/providers.py +++ b/automata/core/agent/providers.py @@ -45,7 +45,7 @@ def __init__(self, instructions: str, config: AutomataOpenAIAgentConfig) -> None super().__init__(instructions) self.config = config self.iteration_count = 0 - self.agent_conversations = OpenAIConversation() + self.agent_conversation_database = OpenAIConversation() self.completed = False self._setup() @@ -133,7 +133,7 @@ def run(self) -> str: except AgentStopIteration: break - last_message = self.agent_conversations.get_latest_message() + last_message = self.agent_conversation_database.get_latest_message() if self.iteration_count >= self.config.max_iterations: raise AgentMaxIterError("The agent exceeded the maximum number of iterations.") if not self.completed or not isinstance(last_message, OpenAIChatMessage): @@ -148,7 +148,7 @@ def set_database_provider(self, provider: LLMConversationDatabaseProvider) -> No if self.database_provider: raise AgentDatabaseError("The database provider has already been set.") self.database_provider = provider - self.agent_conversations.register_observer(provider) + self.agent_conversation_database.register_observer(provider) def _build_initial_messages( self, instruction_formatter: Dict[str, str] @@ -217,21 +217,21 @@ def _setup(self) -> None: AgentError: If the agent fails to initialize. """ logger.debug(f"Setting up agent with tools = {self.config.tools}") - self.agent_conversations.add_message( + self.agent_conversation_database.add_message( OpenAIChatMessage(role="system", content=self.config.system_instruction) ) for message in list( self._build_initial_messages({"user_input_instructions": self.instructions}) ): logger.debug(f"Adding the following initial mesasge to the conversation {message}") - self.agent_conversations.add_message(message) + self.agent_conversation_database.add_message(message) logging.debug(f"\n{('-' * 120)}") self.chat_provider = OpenAIChatCompletionProvider( model=self.config.model, temperature=self.config.temperature, stream=self.config.stream, - conversation=self.agent_conversations, + conversation=self.agent_conversation_database, functions=self.functions, ) self._initialized = True diff --git a/automata/core/agent/tool/tool_utils.py b/automata/core/agent/tool/tool_utils.py index bd8c3352..8fc5b97a 100644 --- a/automata/core/agent/tool/tool_utils.py +++ b/automata/core/agent/tool/tool_utils.py @@ -6,8 +6,8 @@ from automata.config.base import ConfigCategory, LLMProvider from automata.core.agent.error import AgentGeneralError, UnknownToolError from automata.core.base.agent import AgentToolProviders -from automata.core.base.database.vector import JSONEmbeddingVectorDatabase from automata.core.base.singleton import Singleton +from automata.core.base.symbol_embedding import JSONSymbolEmbeddingVectorDatabase from automata.core.base.tool import Tool from automata.core.coding.py.reader import PyReader from automata.core.coding.py.writer import PyWriter @@ -24,7 +24,7 @@ from automata.core.symbol.graph import SymbolGraph from automata.core.symbol.search.rank import SymbolRank, SymbolRankConfig from automata.core.symbol.search.symbol_search import SymbolSearch -from automata.core.symbol_embedding.embedding_builders import SymbolDocEmbeddingBuilder +from automata.core.symbol_embedding.builders import SymbolDocEmbeddingBuilder from automata.core.symbol_embedding.similarity import SymbolSimilarityCalculator from automata.core.utils import get_config_fpath @@ -154,7 +154,7 @@ def create_symbol_code_similarity(self) -> SymbolSimilarityCalculator: code_embedding_fpath = self.overrides.get( "code_embedding_fpath", DependencyFactory.DEFAULT_CODE_EMBEDDING_FPATH ) - code_embedding_db = JSONEmbeddingVectorDatabase(code_embedding_fpath) + code_embedding_db = JSONSymbolEmbeddingVectorDatabase(code_embedding_fpath) embedding_provider = self.overrides.get("embedding_provider", OpenAIEmbeddingProvider()) code_embedding_handler = SymbolCodeEmbeddingHandler(code_embedding_db, embedding_provider) @@ -170,7 +170,7 @@ def create_symbol_doc_similarity(self) -> SymbolSimilarityCalculator: doc_embedding_fpath = self.overrides.get( "doc_embedding_fpath", DependencyFactory.DEFAULT_DOC_EMBEDDING_FPATH ) - doc_embedding_db = JSONEmbeddingVectorDatabase(doc_embedding_fpath) + doc_embedding_db = JSONSymbolEmbeddingVectorDatabase(doc_embedding_fpath) embedding_provider = self.overrides.get("embedding_provider", OpenAIEmbeddingProvider()) symbol_search = self.get("symbol_search") diff --git a/automata/core/base/database/vector.py b/automata/core/base/database/vector.py index 8abe1f84..d96419cc 100644 --- a/automata/core/base/database/vector.py +++ b/automata/core/base/database/vector.py @@ -5,8 +5,6 @@ import jsonpickle -from automata.core.symbol_embedding.base import SymbolEmbedding - logger = logging.getLogger(__name__) T = TypeVar("T") @@ -125,77 +123,3 @@ def get(self, key: K) -> T: def clear(self): self.data = [] self.index = {} - - -class JSONEmbeddingVectorDatabase(JSONVectorDatabase): - """Concrete class to provide a vector database that saves into a JSON file.""" - - def __init__(self, file_path: str): - super().__init__(file_path) - - def entry_to_key(self, entry: SymbolEmbedding) -> str: - """Method to generate a hashable key from an entry of type T.""" - return entry.symbol.dotpath - - def get_all_entries(self) -> List[SymbolEmbedding]: - return sorted(self.data, key=lambda x: self.entry_to_key(x)) - - -# class JSONEmbeddingVectorDatabase(VectorDatabaseProvider): -# """Concrete class to provide a vector database that saves into a JSON file.""" - -# def __init__(self, file_path: str): -# self.file_path = file_path -# self.data: List[SymbolEmbedding] = [] -# self.index: Dict[str, int] = {} -# self.load() - -# def save(self): -# """Saves the vector database to the JSON file.""" -# with open(self.file_path, "w") as file: -# encoded_data = jsonpickle.encode(self.data) -# file.write(encoded_data) - -# def load(self): -# """Loads the vector database from the JSON file.""" -# try: -# with open(self.file_path, "r") as file: -# self.data = jsonpickle.decode(file.read()) -# # We index on the dotpath of the symbol, which is unique and indepenent of commit hash -# self.index = {embedding.symbol.dotpath: i for i, embedding in enumerate(self.data)} -# except FileNotFoundError: -# logger.info(f"Creating new vector embedding db at {self.file_path}") - -# def add(self, embedding: SymbolEmbedding): -# self.data.append(embedding) -# self.index[embedding.symbol.dotpath] = len(self.data) - 1 - -# def update_database(self, embedding: SymbolEmbedding): -# if embedding.symbol not in self.index: -# raise KeyError(f"Symbol {embedding.symbol} not in database") -# self.data[self.index[embedding.symbol.dotpath]] = embedding - -# def discard(self, symbol: Symbol): -# if symbol.dotpath not in self.index: -# raise KeyError(f"Symbol {symbol} not in database") -# index = self.index[symbol.dotpath] -# del self.data[index] -# del self.index[symbol.dotpath] -# # Recalculate indices after deletion -# self.index = {embedding.symbol.dotpath: i for i, embedding in enumerate(self.data)} - -# def contains(self, symbol: Symbol) -> bool: -# return symbol.dotpath in self.index - -# def get(self, symbol: Symbol) -> SymbolEmbedding: -# if symbol.dotpath not in self.index: -# raise KeyError(f"Symbol {symbol} not in database") -# return self.data[self.index[symbol.dotpath]] - -# def clear(self): -# self.data = [] -# self.index = {} - -# def get_all_entries(self) -> List[Symbol]: -# symbol_list = [embedding.symbol for embedding in self.data] -# return sorted(symbol_list, key=lambda x: str(x.dotpath)) diff --git a/automata/core/base/embedding.py b/automata/core/base/embedding.py new file mode 100644 index 00000000..8c6e4299 --- /dev/null +++ b/automata/core/base/embedding.py @@ -0,0 +1,51 @@ +import abc +import logging +from enum import Enum +from typing import Dict + +import numpy as np + +from automata.core.base.symbol import Symbol + +logger = logging.getLogger(__name__) + + +class EmbeddingNormType(Enum): + L1 = "l1" + L2 = "l2" + SOFTMAX = "softmax" + + +class EmbeddingProvider(abc.ABC): + """A class to provide embeddings for symbols""" + + @abc.abstractmethod + def build_embedding_array(self, symbol_source: str) -> np.ndarray: + pass + + +class EmbeddingSimilarityCalculator(abc.ABC): + @abc.abstractmethod + def calculate_query_similarity_dict(self, query_text: str) -> Dict[Symbol, float]: + """An abstract method to get the similarity between a query and all symbols""" + pass + + @abc.abstractmethod + def _calculate_embedding_similarity(self, embedding_array: np.ndarray) -> np.ndarray: + """An abstract method to calculate the similarity between the embedding array target embeddings.""" + pass + + @staticmethod + def _normalize_embeddings( + embeddings_array: np.ndarray, norm_type: EmbeddingNormType + ) -> np.ndarray: + if norm_type == EmbeddingNormType.L1: + norm = np.sum(np.abs(embeddings_array), axis=1, keepdims=True) + return embeddings_array / norm + elif norm_type == EmbeddingNormType.L2: + return embeddings_array / np.linalg.norm(embeddings_array, axis=1, keepdims=True) + elif norm_type == EmbeddingNormType.SOFTMAX: + e_x = np.exp(embeddings_array - np.max(embeddings_array, axis=1, keepdims=True)) + return e_x / np.sum(e_x, axis=1, keepdims=True) + else: + raise ValueError(f"Invalid normalization type {norm_type}") diff --git a/automata/core/symbol/scip_pb2.py b/automata/core/base/scip_pb2.py similarity index 100% rename from automata/core/symbol/scip_pb2.py rename to automata/core/base/scip_pb2.py diff --git a/automata/core/symbol/base.py b/automata/core/base/symbol.py similarity index 99% rename from automata/core/symbol/base.py rename to automata/core/base/symbol.py index 23f95e62..684c3a66 100644 --- a/automata/core/symbol/base.py +++ b/automata/core/base/symbol.py @@ -3,7 +3,7 @@ from enum import Enum from typing import Any, Dict, Optional, Tuple -from automata.core.symbol.scip_pb2 import Descriptor as DescriptorProto # type: ignore +from automata.core.base.scip_pb2 import Descriptor as DescriptorProto # type: ignore class SymbolDescriptor: diff --git a/automata/core/base/symbol_embedding.py b/automata/core/base/symbol_embedding.py new file mode 100644 index 00000000..7d348d7c --- /dev/null +++ b/automata/core/base/symbol_embedding.py @@ -0,0 +1,117 @@ +import abc +from typing import Any, List, Optional + +import numpy as np + +from automata.core.base.database.vector import JSONVectorDatabase +from automata.core.base.embedding import EmbeddingProvider +from automata.core.base.symbol import Symbol + + +class SymbolEmbedding(abc.ABC): + """Abstract base class for different types of embeddings""" + + def __init__(self, symbol: Symbol, embedding_source: str, vector: np.ndarray): + self.symbol = symbol + self.embedding_source = embedding_source + self.vector = vector + + @abc.abstractmethod + def __str__(self) -> str: + pass + + +class JSONSymbolEmbeddingVectorDatabase(JSONVectorDatabase): + """Concrete class to provide a vector database that saves into a JSON file.""" + + def __init__(self, file_path: str): + super().__init__(file_path) + + def entry_to_key(self, entry: SymbolEmbedding) -> str: + """Method to generate a hashable key from an entry of type T.""" + return entry.symbol.dotpath + + def get_all_entries(self) -> List[SymbolEmbedding]: + return sorted(self.data, key=lambda x: self.entry_to_key(x)) + + +class SymbolCodeEmbedding(SymbolEmbedding): + """A concrete class for symbol code embeddings""" + + def __init__(self, symbol: Symbol, source_code: str, vector: np.ndarray): + super().__init__(symbol, source_code, vector) + + def __str__(self) -> str: + return f"SymbolCodeEmbedding(\nsymbol={self.symbol},\n\nembedding_source={self.embedding_source}\n\nvector_length={len(self.vector)}\n)" + + +class SymbolDocEmbedding(SymbolEmbedding): + """A concrete class for symbol document embeddings""" + + def __init__( + self, + symbol: Symbol, + document: str, + vector: np.ndarray, + source_code: Optional[str] = None, + summary: Optional[str] = None, + context: Optional[str] = None, + ) -> None: + super().__init__(symbol, document, vector) + # begin additional meta data + self.source_code = source_code + self.summary = summary + self.context = context + + def __str__(self) -> str: + return f"SymbolDocEmbedding(\nsymbol={self.symbol},\n\nembedding_source={self.embedding_source}\n\nvector_length={len(self.vector)}\n\nsource_code={self.source_code}\n\nsummary={self.summary}\n\ncontext={self.context}\n)" + + +class SymbolEmbeddingBuilder(abc.ABC): + """An abstract class to build embeddings for symbols""" + + def __init__( + self, + embedding_provider: EmbeddingProvider, + ) -> None: + self.embedding_provider = embedding_provider + + @abc.abstractmethod + def build(self, source_text: str, symbol: Symbol) -> Any: + """An abstract method to build the embedding for a symbol""" + pass + + def fetch_embedding_context(self, symbol: Symbol) -> str: + """For a code embedding the context is the source code itself.""" + from automata.core.symbol.symbol_utils import ( # imported late for mocking + convert_to_fst_object, + ) + + return str(convert_to_fst_object(symbol)) + + +class SymbolEmbeddingHandler(abc.ABC): + """An abstract class to handle the embedding of symbols""" + + @abc.abstractmethod + def __init__( + self, + embedding_db: JSONSymbolEmbeddingVectorDatabase, + embedding_builder: SymbolEmbeddingBuilder, + ) -> None: + """An abstract constructor for SymbolEmbeddingHandler""" + self.embedding_db = embedding_db + self.embedding_builder = embedding_builder + + @abc.abstractmethod + def get_embedding(self, symbol: Symbol) -> Any: + """An abstract method to get the embedding for a symbol""" + pass + + @abc.abstractmethod + def process_embedding(self, symbol: Symbol) -> None: + """An abstract method to process the embedding for a symbol""" + pass + + def get_all_supported_symbols(self) -> List[Symbol]: + return [embedding.symbol for embedding in self.embedding_db.get_all_entries()] diff --git a/automata/core/coding/py/writer.py b/automata/core/coding/py/writer.py index af160827..23de56e8 100644 --- a/automata/core/coding/py/writer.py +++ b/automata/core/coding/py/writer.py @@ -8,6 +8,8 @@ import pypandoc from redbaron import ClassNode, DefNode, Node, NodeList, RedBaron +from automata.core.base.symbol import Symbol +from automata.core.base.symbol_embedding import SymbolDocEmbedding from automata.core.coding.directory import DirectoryManager from automata.core.coding.py.module_loader import py_module_loader from automata.core.coding.py.navigation import ( @@ -17,8 +19,6 @@ find_syntax_tree_node, ) from automata.core.coding.py.reader import PyReader -from automata.core.symbol.base import Symbol -from automata.core.symbol_embedding.base import SymbolDocEmbedding logger = logging.getLogger(__name__) diff --git a/automata/core/context/py/retriever.py b/automata/core/context/py/retriever.py index cba3f44a..61bdb474 100644 --- a/automata/core/context/py/retriever.py +++ b/automata/core/context/py/retriever.py @@ -7,8 +7,8 @@ from redbaron import RedBaron from automata.core.base.database.vector import VectorDatabaseProvider +from automata.core.base.symbol import Symbol from automata.core.coding.py.reader import PyReader -from automata.core.symbol.base import Symbol from automata.core.symbol.graph import SymbolGraph from automata.core.symbol.symbol_utils import ( convert_to_fst_object, diff --git a/automata/core/llm/embedding.py b/automata/core/llm/embedding.py deleted file mode 100644 index 51355e86..00000000 --- a/automata/core/llm/embedding.py +++ /dev/null @@ -1,101 +0,0 @@ -import abc -import logging -from enum import Enum -from typing import Any, Dict, List - -import numpy as np - -from automata.core.base.database.vector import VectorDatabaseProvider -from automata.core.symbol.base import Symbol - -logger = logging.getLogger(__name__) - - -class EmbeddingNormType(Enum): - L1 = "l1" - L2 = "l2" - SOFTMAX = "softmax" - - -class EmbeddingProvider(abc.ABC): - """A class to provide embeddings for symbols""" - - @abc.abstractmethod - def build_embedding_array(self, symbol_source: str) -> np.ndarray: - pass - - -class SymbolEmbeddingBuilder(abc.ABC): - """An abstract class to build embeddings for symbols""" - - @abc.abstractmethod - def __init__(self, embedding_provider: EmbeddingProvider) -> None: - """An abstract constructor for SymbolEmbeddingBuilder""" - self.embedding_provider = embedding_provider - - @abc.abstractmethod - def build(self, source_text: str, symbol: Symbol) -> Any: - """An abstract method to build the embedding for a symbol""" - pass - - def fetch_embedding_context(self, symbol: Symbol) -> str: - """For a code embedding the context is the source code itself.""" - from automata.core.symbol.symbol_utils import ( # imported late for mocking - convert_to_fst_object, - ) - - return str(convert_to_fst_object(symbol)) - - -class SymbolEmbeddingHandler(abc.ABC): - """An abstract class to handle the embedding of symbols""" - - @abc.abstractmethod - def __init__( - self, - embedding_db: VectorDatabaseProvider, - embedding_builder: SymbolEmbeddingBuilder, - ) -> None: - """An abstract constructor for SymbolEmbeddingHandler""" - self.embedding_db = embedding_db - self.embedding_builder = embedding_builder - - @abc.abstractmethod - def get_embedding(self, symbol: Symbol) -> Any: - """An abstract method to get the embedding for a symbol""" - pass - - @abc.abstractmethod - def process_embedding(self, symbol: Symbol) -> None: - """An abstract method to process the embedding for a symbol""" - pass - - def get_all_supported_symbols(self) -> List[Symbol]: - return [embedding.symbol for embedding in self.embedding_db.get_all_entries()] - - -class EmbeddingSimilarityCalculator(abc.ABC): - @abc.abstractmethod - def calculate_query_similarity_dict(self, query_text: str) -> Dict[Symbol, float]: - """An abstract method to get the similarity between a query and all symbols""" - pass - - @abc.abstractmethod - def _calculate_embedding_similarity(self, embedding_array: np.ndarray) -> np.ndarray: - """An abstract method to calculate the similarity between the embedding array target embeddings.""" - pass - - @staticmethod - def _normalize_embeddings( - embeddings_array: np.ndarray, norm_type: EmbeddingNormType - ) -> np.ndarray: - if norm_type == EmbeddingNormType.L1: - norm = np.sum(np.abs(embeddings_array), axis=1, keepdims=True) - return embeddings_array / norm - elif norm_type == EmbeddingNormType.L2: - return embeddings_array / np.linalg.norm(embeddings_array, axis=1, keepdims=True) - elif norm_type == EmbeddingNormType.SOFTMAX: - e_x = np.exp(embeddings_array - np.max(embeddings_array, axis=1, keepdims=True)) - return e_x / np.sum(e_x, axis=1, keepdims=True) - else: - raise ValueError(f"Invalid normalization type {norm_type}") diff --git a/automata/core/llm/providers/openai.py b/automata/core/llm/providers/openai.py index f8de7890..0caaf17b 100644 --- a/automata/core/llm/providers/openai.py +++ b/automata/core/llm/providers/openai.py @@ -7,6 +7,7 @@ import tiktoken from termcolor import colored +from automata.core.base.embedding import EmbeddingProvider from automata.core.base.tool import Tool from automata.core.llm.completion import ( LLMChatCompletionProvider, @@ -14,7 +15,6 @@ LLMCompletionResult, LLMConversation, ) -from automata.core.llm.embedding import EmbeddingProvider from automata.core.utils import set_openai_api_key logger = logging.getLogger(__name__) @@ -218,7 +218,7 @@ def __init__( self.temperature = temperature self.stream = stream self.functions = functions - self.agent_conversations = conversation + self.agent_conversation_database = conversation set_openai_api_key() def get_next_assistant_completion(self) -> OpenAIChatMessage: @@ -229,7 +229,7 @@ def get_next_assistant_completion(self) -> OpenAIChatMessage: if functions: response = openai.ChatCompletion.create( model=self.model, - messages=self.agent_conversations.get_messages_for_next_completion(), + messages=self.agent_conversation_database.get_messages_for_next_completion(), functions=functions, function_call="auto", # auto is default, but we'll be explicit stream=self.stream, @@ -237,7 +237,7 @@ def get_next_assistant_completion(self) -> OpenAIChatMessage: else: response = openai.ChatCompletion.create( model=self.model, - messages=self.agent_conversations.get_messages_for_next_completion(), + messages=self.agent_conversation_database.get_messages_for_next_completion(), stream=self.stream, ) if self.stream: @@ -249,11 +249,11 @@ def get_next_assistant_completion(self) -> OpenAIChatMessage: ) def reset(self) -> None: - self.agent_conversations.reset_conversation() + self.agent_conversation_database.reset_conversation() def standalone_call(self, prompt: str) -> str: """Return the completion message based on the provided prompt.""" - if self.agent_conversations.messages: + if self.agent_conversation_database.messages: raise ValueError( "The conversation is not empty. Please call reset() before calling standalone_call()." ) @@ -266,11 +266,11 @@ def standalone_call(self, prompt: str) -> str: def add_message(self, message: LLMChatMessage) -> None: if not isinstance(message, OpenAIChatMessage): - self.agent_conversations.add_message( + self.agent_conversation_database.add_message( OpenAIChatMessage(role=message.role, content=message.content) ) else: - self.agent_conversations.add_message(message) + self.agent_conversation_database.add_message(message) logger.debug( f"Approximately {self.get_approximate_tokens_consumed()} tokens were after adding the latest message." ) @@ -361,7 +361,7 @@ def get_approximate_tokens_consumed(self) -> int: "\n".join( [ json.dumps(ele) - for ele in self.agent_conversations.get_messages_for_next_completion() + for ele in self.agent_conversation_database.get_messages_for_next_completion() ] ) ) diff --git a/automata/core/memory_store/agent_conversations.py b/automata/core/memory_store/agent_conversation_database.py similarity index 93% rename from automata/core/memory_store/agent_conversations.py rename to automata/core/memory_store/agent_conversation_database.py index 5a2b42f0..b284bfc7 100644 --- a/automata/core/memory_store/agent_conversations.py +++ b/automata/core/memory_store/agent_conversation_database.py @@ -1,7 +1,6 @@ from typing import List from automata.config import CONVERSATION_DB_PATH -from automata.core.agent.error import AgentDatabaseError from automata.core.llm.completion import LLMChatMessage, LLMConversationDatabaseProvider @@ -32,7 +31,7 @@ def last_interaction_id(self) -> int: def save_message(self, message: LLMChatMessage) -> None: """TODO - Think about how to handle function calls, e.g. OpenAIChatMessage, and other chat message providers""" if self.session_id is None: - raise AgentDatabaseError("The database session_id has not been set.") + raise ValueError("The database session_id has not been set.") interaction_id = self.last_interaction_id + 1 interaction = { "role": message.role, diff --git a/automata/core/memory_store/symbol_code_embedding.py b/automata/core/memory_store/symbol_code_embedding.py index 6febb733..d37610c8 100644 --- a/automata/core/memory_store/symbol_code_embedding.py +++ b/automata/core/memory_store/symbol_code_embedding.py @@ -1,10 +1,12 @@ import logging -from automata.core.base.database.vector import VectorDatabaseProvider -from automata.core.llm.embedding import SymbolEmbeddingHandler -from automata.core.symbol.base import Symbol -from automata.core.symbol_embedding.base import SymbolCodeEmbedding -from automata.core.symbol_embedding.embedding_builders import SymbolCodeEmbeddingBuilder +from automata.core.base.symbol import Symbol +from automata.core.base.symbol_embedding import ( + JSONSymbolEmbeddingVectorDatabase, + SymbolCodeEmbedding, + SymbolEmbeddingHandler, +) +from automata.core.symbol_embedding.builders import SymbolCodeEmbeddingBuilder logger = logging.getLogger(__name__) @@ -14,7 +16,7 @@ class SymbolCodeEmbeddingHandler(SymbolEmbeddingHandler): def __init__( self, - embedding_db: VectorDatabaseProvider, + embedding_db: JSONSymbolEmbeddingVectorDatabase, embedding_builder: SymbolCodeEmbeddingBuilder, ) -> None: super().__init__(embedding_db, embedding_builder) diff --git a/automata/core/memory_store/symbol_doc_embedding.py b/automata/core/memory_store/symbol_doc_embedding.py index 23ea5edf..e294ba41 100644 --- a/automata/core/memory_store/symbol_doc_embedding.py +++ b/automata/core/memory_store/symbol_doc_embedding.py @@ -1,10 +1,12 @@ import logging -from automata.core.base.database.vector import VectorDatabaseProvider -from automata.core.llm.embedding import SymbolEmbeddingHandler -from automata.core.symbol.base import Symbol -from automata.core.symbol_embedding.base import SymbolDocEmbedding -from automata.core.symbol_embedding.embedding_builders import SymbolDocEmbeddingBuilder +from automata.core.base.symbol import Symbol +from automata.core.base.symbol_embedding import ( + JSONSymbolEmbeddingVectorDatabase, + SymbolDocEmbedding, + SymbolEmbeddingHandler, +) +from automata.core.symbol_embedding.builders import SymbolDocEmbeddingBuilder logger = logging.getLogger(__name__) @@ -14,7 +16,7 @@ class SymbolDocEmbeddingHandler(SymbolEmbeddingHandler): def __init__( self, - embedding_db: VectorDatabaseProvider, + embedding_db: JSONSymbolEmbeddingVectorDatabase, embedding_builder: SymbolDocEmbeddingBuilder, ) -> None: self.embedding_db = embedding_db diff --git a/automata/core/symbol/graph.py b/automata/core/symbol/graph.py index b713fa57..d828d7a5 100644 --- a/automata/core/symbol/graph.py +++ b/automata/core/symbol/graph.py @@ -10,15 +10,15 @@ from tqdm import tqdm from automata.config import MAX_WORKERS -from automata.core.coding.py.module_loader import py_module_loader -from automata.core.symbol.base import ( +from automata.core.base.scip_pb2 import Index, SymbolRole # type: ignore +from automata.core.base.symbol import ( Symbol, SymbolDescriptor, SymbolFile, SymbolReference, ) +from automata.core.coding.py.module_loader import py_module_loader from automata.core.symbol.parser import parse_symbol -from automata.core.symbol.scip_pb2 import Index, SymbolRole # type: ignore from automata.core.symbol.symbol_utils import ( convert_to_fst_object, get_rankable_symbols, diff --git a/automata/core/symbol/parser.py b/automata/core/symbol/parser.py index 0329d0d1..9ec3d835 100644 --- a/automata/core/symbol/parser.py +++ b/automata/core/symbol/parser.py @@ -1,7 +1,7 @@ import re from typing import List, Optional -from automata.core.symbol.base import Symbol, SymbolDescriptor, SymbolPackage +from automata.core.base.symbol import Symbol, SymbolDescriptor, SymbolPackage class _SymbolParser: diff --git a/automata/core/symbol/search/rank.py b/automata/core/symbol/search/rank.py index 7299139b..502bc0d4 100644 --- a/automata/core/symbol/search/rank.py +++ b/automata/core/symbol/search/rank.py @@ -4,7 +4,7 @@ from networkx.exception import NetworkXError from pydantic import BaseModel -from automata.core.symbol.base import Symbol +from automata.core.base.symbol import Symbol class SymbolRankConfig(BaseModel): diff --git a/automata/core/symbol/search/symbol_search.py b/automata/core/symbol/search/symbol_search.py index a4162521..43e1110b 100644 --- a/automata/core/symbol/search/symbol_search.py +++ b/automata/core/symbol/search/symbol_search.py @@ -4,8 +4,8 @@ import networkx as nx import numpy as np +from automata.core.base.symbol import Symbol, SymbolReference from automata.core.coding.py.module_loader import py_module_loader -from automata.core.symbol.base import Symbol, SymbolReference from automata.core.symbol.graph import SymbolGraph from automata.core.symbol.parser import parse_symbol from automata.core.symbol.search.rank import SymbolRank, SymbolRankConfig diff --git a/automata/core/symbol/symbol_utils.py b/automata/core/symbol/symbol_utils.py index caf55b06..99656982 100644 --- a/automata/core/symbol/symbol_utils.py +++ b/automata/core/symbol/symbol_utils.py @@ -2,8 +2,8 @@ from redbaron import RedBaron +from automata.core.base.symbol import Symbol, SymbolDescriptor from automata.core.coding.py.module_loader import py_module_loader -from automata.core.symbol.base import Symbol, SymbolDescriptor def convert_to_fst_object(symbol: Symbol) -> RedBaron: diff --git a/automata/core/symbol_embedding/base.py b/automata/core/symbol_embedding/base.py deleted file mode 100644 index 602e1253..00000000 --- a/automata/core/symbol_embedding/base.py +++ /dev/null @@ -1,51 +0,0 @@ -import abc -from typing import Optional - -import numpy as np - -from automata.core.symbol.base import Symbol - - -class SymbolEmbedding(abc.ABC): - """Abstract base class for different types of embeddings""" - - def __init__(self, symbol: Symbol, embedding_source: str, vector: np.ndarray): - self.symbol = symbol - self.embedding_source = embedding_source - self.vector = vector - - @abc.abstractmethod - def __str__(self) -> str: - pass - - -class SymbolCodeEmbedding(SymbolEmbedding): - """A concrete class for symbol code embeddings""" - - def __init__(self, symbol: Symbol, source_code: str, vector: np.ndarray): - super().__init__(symbol, source_code, vector) - - def __str__(self) -> str: - return f"SymbolCodeEmbedding(\nsymbol={self.symbol},\n\nembedding_source={self.embedding_source}\n\nvector_length={len(self.vector)}\n)" - - -class SymbolDocEmbedding(SymbolEmbedding): - """A concrete class for symbol document embeddings""" - - def __init__( - self, - symbol: Symbol, - document: str, - vector: np.ndarray, - source_code: Optional[str] = None, - summary: Optional[str] = None, - context: Optional[str] = None, - ) -> None: - super().__init__(symbol, document, vector) - # begin additional meta data - self.source_code = source_code - self.summary = summary - self.context = context - - def __str__(self) -> str: - return f"SymbolDocEmbedding(\nsymbol={self.symbol},\n\nembedding_source={self.embedding_source}\n\nvector_length={len(self.vector)}\n\nsource_code={self.source_code}\n\nsummary={self.summary}\n\ncontext={self.context}\n)" diff --git a/automata/core/symbol_embedding/embedding_builders.py b/automata/core/symbol_embedding/builders.py similarity index 94% rename from automata/core/symbol_embedding/embedding_builders.py rename to automata/core/symbol_embedding/builders.py index cfcca326..e465ef41 100644 --- a/automata/core/symbol_embedding/embedding_builders.py +++ b/automata/core/symbol_embedding/builders.py @@ -3,23 +3,21 @@ from jinja2 import Template from automata.config.prompt.doc_generation import DEFAULT_DOC_GENERATION_PROMPT +from automata.core.base.embedding import EmbeddingProvider +from automata.core.base.symbol import Symbol +from automata.core.base.symbol_embedding import ( + SymbolCodeEmbedding, + SymbolDocEmbedding, + SymbolEmbeddingBuilder, +) from automata.core.context.py.retriever import PyContextRetriever from automata.core.llm.completion import LLMChatCompletionProvider -from automata.core.llm.embedding import EmbeddingProvider, SymbolEmbeddingBuilder -from automata.core.symbol.base import Symbol from automata.core.symbol.search.symbol_search import SymbolSearch -from automata.core.symbol_embedding.base import SymbolCodeEmbedding, SymbolDocEmbedding class SymbolCodeEmbeddingBuilder(SymbolEmbeddingBuilder): """Builds `Symbol` source code embeddings.""" - def __init__( - self, - embedding_provider: EmbeddingProvider, - ) -> None: - self.embedding_provider = embedding_provider - def build(self, source_code: str, symbol: Symbol) -> SymbolCodeEmbedding: embedding_vector = self.embedding_provider.build_embedding_array(source_code) return SymbolCodeEmbedding(symbol, source_code, embedding_vector) diff --git a/automata/core/symbol_embedding/similarity.py b/automata/core/symbol_embedding/similarity.py index c6337f70..50455752 100644 --- a/automata/core/symbol_embedding/similarity.py +++ b/automata/core/symbol_embedding/similarity.py @@ -3,13 +3,13 @@ import numpy as np -from automata.core.llm.embedding import ( +from automata.core.base.embedding import ( EmbeddingNormType, EmbeddingProvider, EmbeddingSimilarityCalculator, - SymbolEmbeddingHandler, ) -from automata.core.symbol.base import Symbol +from automata.core.base.symbol import Symbol +from automata.core.base.symbol_embedding import SymbolEmbeddingHandler logger = logging.getLogger(__name__) diff --git a/automata/docs/core/agent/automata_memory_database.rst b/automata/docs/core/agent/automata_memory_database.rst index 63ca80ad..29441deb 100644 --- a/automata/docs/core/agent/automata_memory_database.rst +++ b/automata/docs/core/agent/automata_memory_database.rst @@ -9,7 +9,7 @@ allowing users to get and put messages for a specific session. Related Symbols --------------- -- ``automata.core.symbol.base.Symbol`` +- ``automata.core.base.symbol.Symbol`` - ``automata.core.agent.agent.AutomataAgent`` - ``config.config_types.AgentConfigName`` - ``automata.core.database.vector.JSONEmbeddingVectorDatabase`` diff --git a/automata/docs/core/base/database/json_embedding_vector_database.rst b/automata/docs/core/base/database/json_embedding_vector_database.rst index 9f6e3f29..ef9a0f29 100644 --- a/automata/docs/core/base/database/json_embedding_vector_database.rst +++ b/automata/docs/core/base/database/json_embedding_vector_database.rst @@ -21,8 +21,8 @@ Related Symbols - ``JSONEmbeddingVectorDatabase`` inherits from ``automata.core.base.database.vector.VectorDatabaseProvider`` -- ``automata.core.symbol.base.Symbol`` -- ``automata.core.symbol_embedding.base.SymbolEmbedding`` +- ``automata.core.base.symbol.Symbol`` +- ``automata.core.base.symbol_embedding.SymbolEmbedding`` Example ------- @@ -34,7 +34,7 @@ and retrieving vectors associated with symbols. .. code:: python from automata.core.base.database.vector import JSONEmbeddingVectorDatabase - from automata.core.symbol.base import Symbol, SymbolEmbedding + from automata.core.base.symbol import Symbol, SymbolEmbedding file_path = "path/to/json/database.json" vector_db = JSONEmbeddingVectorDatabase(file_path) diff --git a/automata/docs/core/base/database/json_vector_database.rst b/automata/docs/core/base/database/json_vector_database.rst index 42333bbc..fc96cd04 100644 --- a/automata/docs/core/base/database/json_vector_database.rst +++ b/automata/docs/core/base/database/json_vector_database.rst @@ -35,7 +35,7 @@ The following is an example demonstrating usage of the .. code:: python from automata.core.base.database.vector import JSONVectorDatabase - from automata.core.symbol.base import SymbolEmbedding + from automata.core.base.symbol import SymbolEmbedding # Creating an instance of JSONVectorDatabase vector_db = JSONVectorDatabase("path_to_json_file.json") diff --git a/automata/docs/core/base/database/symbol_database_provider.rst b/automata/docs/core/base/database/symbol_database_provider.rst index 40558ea8..d1f6dc16 100644 --- a/automata/docs/core/base/database/symbol_database_provider.rst +++ b/automata/docs/core/base/database/symbol_database_provider.rst @@ -9,8 +9,8 @@ embeddings in the database. Related Symbols --------------- -- ``automata.core.symbol.base.Symbol`` -- ``automata.core.symbol_embedding.base.SymbolEmbedding`` +- ``automata.core.base.symbol.Symbol`` +- ``automata.core.base.symbol_embedding.SymbolEmbedding`` - ``automata.core.memory_store.symbol_code_embedding.SymbolCodeEmbeddingHandler`` - ``automata.core.base.database.vector.JSONEmbeddingVectorDatabase`` - ``automata.tests.utils.factories.symbol_search_live`` @@ -24,7 +24,7 @@ The following is an example of a custom implementation of .. code:: python from automata.core.base.database.provider import SymbolDatabaseProvider - from automata.core.symbol.base import Symbol, SymbolEmbedding + from automata.core.base.symbol import Symbol, SymbolEmbedding class CustomDatabase(SymbolDatabaseProvider): def __init__(self): diff --git a/automata/docs/core/base/database/vector_database_provider.rst b/automata/docs/core/base/database/vector_database_provider.rst index 88f80a42..5ff22bb2 100644 --- a/automata/docs/core/base/database/vector_database_provider.rst +++ b/automata/docs/core/base/database/vector_database_provider.rst @@ -25,8 +25,8 @@ Related Symbols - ``automata.core.base.database.vector.JSONEmbeddingVectorDatabase`` - ``automata.core.memory_store.symbol_code_embedding.SymbolCodeEmbeddingHandler`` - ``automata.core.base.database.provider.SymbolDatabaseProvider`` -- ``automata.core.symbol.base.Symbol`` -- ``automata.core.symbol_embedding.base.SymbolEmbedding`` +- ``automata.core.base.symbol.Symbol`` +- ``automata.core.base.symbol_embedding.SymbolEmbedding`` Example ------- diff --git a/automata/docs/core/base/singleton.rst b/automata/docs/core/base/singleton.rst index 0fc7ce84..85341cef 100644 --- a/automata/docs/core/base/singleton.rst +++ b/automata/docs/core/base/singleton.rst @@ -16,7 +16,7 @@ Related Symbols - ``automata.tests.unit.sample_modules.sample.OuterClass.InnerClass`` - ``automata.core.llm.completion.LLMChatCompletionProvider.__init__`` - ``automata.tests.conftest.MockRepositoryManager`` -- ``automata.core.symbol.base.Symbol`` +- ``automata.core.base.symbol.Symbol`` - ``automata.tests.unit.test_py_reader.getter`` - ``automata.core.coding.py.module_loader.PyModuleLoader.__init__`` diff --git a/automata/docs/core/coding/file.rst b/automata/docs/core/coding/file.rst index cba2e836..c4c5a219 100644 --- a/automata/docs/core/coding/file.rst +++ b/automata/docs/core/coding/file.rst @@ -22,13 +22,13 @@ Related Symbols - ``automata.core.coding.directory.Node`` - ``automata.tests.unit.test_symbol_graph.test_get_all_files`` - ``automata.tests.unit.test_database_vector.test_save`` -- ``automata.core.symbol.base.SymbolFile`` +- ``automata.core.base.symbol.SymbolFile`` - ``automata.tests.unit.test_database_vector.test_init_vector`` - ``automata.core.base.database.vector.JSONEmbeddingVectorDatabase`` - ``automata.tests.unit.test_database_vector.test_load`` -- ``automata.core.symbol.base.SymbolFile.__eq__`` +- ``automata.core.base.symbol.SymbolFile.__eq__`` - ``automata.tests.unit.sample_modules.sample.EmptyClass`` -- ``automata.core.symbol.base.SymbolReference`` +- ``automata.core.base.symbol.SymbolReference`` - ``automata.tests.unit.test_task_environment.TestURL`` Usage Example diff --git a/automata/docs/core/coding/py/py_doc_writer.rst b/automata/docs/core/coding/py/py_doc_writer.rst index b53bc83c..badb1645 100644 --- a/automata/docs/core/coding/py/py_doc_writer.rst +++ b/automata/docs/core/coding/py/py_doc_writer.rst @@ -24,8 +24,8 @@ Related Symbols - ``automata.core.coding.py.reader.PyReader`` - ``automata.core.coding.directory.DirectoryManager`` -- ``automata.core.symbol.base.Symbol`` -- ``automata.core.symbol_embedding.base.SymbolDocEmbedding`` +- ``automata.core.base.symbol.Symbol`` +- ``automata.core.base.symbol_embedding.SymbolDocEmbedding`` - ``type: ignore`` Example diff --git a/automata/docs/core/coding/py_coding/py_doc_writer.rst b/automata/docs/core/coding/py_coding/py_doc_writer.rst index 318e7b5b..7c049b12 100644 --- a/automata/docs/core/coding/py_coding/py_doc_writer.rst +++ b/automata/docs/core/coding/py_coding/py_doc_writer.rst @@ -25,7 +25,7 @@ Related Symbols - ``automata.tests.unit.sample_modules.sample.OuterClass.InnerClass`` - ``automata.core.coding.py_coding.writer.PyCodeWriter`` - ``automata.core.coding.py_coding.retriever.PyCodeRetriever`` -- ``automata.core.symbol_embedding.base.SymbolDocEmbedding`` +- ``automata.core.base.symbol_embedding.SymbolDocEmbedding`` Example ------- @@ -37,8 +37,8 @@ structure: .. code:: python from automata.core.coding.py_coding.writer import PyDocWriter - from automata.core.symbol.base import Symbol - from automata.core.symbol_embedding.base import SymbolDocEmbedding + from automata.core.base.symbol import Symbol + from automata.core.base.symbol_embedding import SymbolDocEmbedding # Assuming you have the documentation info in the following variables: # - docs: a dictionary containing the {Symbol: SymbolDocEmbedding} pairs diff --git a/automata/docs/core/coding/py_coding/writer/class_or_function_not_found.rst b/automata/docs/core/coding/py_coding/writer/class_or_function_not_found.rst index 64a4b3b1..107d46e2 100644 --- a/automata/docs/core/coding/py_coding/writer/class_or_function_not_found.rst +++ b/automata/docs/core/coding/py_coding/writer/class_or_function_not_found.rst @@ -12,7 +12,7 @@ Related Symbols - ``automata.core.coding.directory.DirectoryManager`` - ``automata.core.coding.py_coding.navigation.find_all_function_and_class_syntax_tree_nodes`` - ``automata.core.coding.py_coding.retriever.PyCodeRetriever`` -- ``automata.core.symbol.base.Symbol`` +- ``automata.core.base.symbol.Symbol`` - ``automata.core.agent.tools.py_code_writer.PyCodeWriterTool`` - ``automata.core.agent.tools.py_code_retriever.PyCodeRetrieverTool`` - ``automata.core.coding.py_coding.writer.PyDocWriter`` diff --git a/automata/docs/core/coding/py_coding/writer/invalid_arguments.rst b/automata/docs/core/coding/py_coding/writer/invalid_arguments.rst index 4a946984..1dcd2552 100644 --- a/automata/docs/core/coding/py_coding/writer/invalid_arguments.rst +++ b/automata/docs/core/coding/py_coding/writer/invalid_arguments.rst @@ -22,7 +22,7 @@ Related Symbols - ``automata.core.coding.py_coding.writer.PyCodeWriter`` - ``automata.core.agent.tools.py_code_writer.PyCodeWriterTool`` - ``automata.core.coding.py_coding.retriever.PyCodeRetriever`` -- ``automata.core.symbol.base.Symbol`` +- ``automata.core.base.symbol.Symbol`` Example ------- diff --git a/automata/docs/core/context/py/py_context_retriever.rst b/automata/docs/core/context/py/py_context_retriever.rst index e1a5a490..d189de22 100644 --- a/automata/docs/core/context/py/py_context_retriever.rst +++ b/automata/docs/core/context/py/py_context_retriever.rst @@ -30,7 +30,7 @@ Related Symbols - ``automata.core.base.database.vector.VectorDatabaseProvider`` - ``automata.core.symbol.graph.SymbolGraph`` -- ``automata.core.symbol.base.Symbol`` +- ``automata.core.base.symbol.Symbol`` - ``automata.core.symbol.symbol_utils.convert_to_fst_object`` - ``automata.core.symbol.symbol_utils.get_rankable_symbols`` @@ -44,7 +44,7 @@ retrieve the context of a symbol from a given Python project: from automata.core.symbol.graph import SymbolGraph from automata.core.context.py.retriever import PyContextRetriever, PyContextRetrieverConfig - from automata.core.symbol.base import Symbol + from automata.core.base.symbol import Symbol # Load a SymbolGraph for the Python project symbol_graph = SymbolGraph(index_path="path/to/index_proto") diff --git a/automata/docs/core/context/py_context/py_context_retriever.rst b/automata/docs/core/context/py_context/py_context_retriever.rst index feb0a022..f0804d2a 100644 --- a/automata/docs/core/context/py_context/py_context_retriever.rst +++ b/automata/docs/core/context/py_context/py_context_retriever.rst @@ -21,7 +21,7 @@ Import Statements from automata.core.coding.py_coding.retriever import PyCodeRetriever from automata.core.database.vector import VectorDatabaseProvider from automata.core.symbol.graph import SymbolGraph - from automata.core.symbol.base import Symbol + from automata.core.base.symbol import Symbol from automata.core.symbol.symbol_utils import ( convert_to_fst_object, get_rankable_symbols, @@ -43,7 +43,7 @@ Related Symbols - ``automata.core.coding.py_coding.retriever.PyCodeRetriever`` - ``automata.core.database.vector.VectorDatabaseProvider`` - ``automata.core.symbol.graph.SymbolGraph`` -- ``automata.core.symbol.base.Symbol`` +- ``automata.core.base.symbol.Symbol`` - ``automata.core.symbol.symbol_utils`` - ``automata.core.utils.root_py_fpath`` @@ -57,7 +57,7 @@ demonstrates its basic functionality. from automata.core.context.py_context.retriever import PyContextRetriever from automata.core.symbol.graph import SymbolGraph - from automata.core.symbol.base import Symbol + from automata.core.base.symbol import Symbol graph = SymbolGraph() symbol = Symbol.from_string("some_string_representation_of_symbol") diff --git a/automata/docs/core/database/json_vector_database.rst b/automata/docs/core/database/json_vector_database.rst index 642c84c2..0d741da3 100644 --- a/automata/docs/core/database/json_vector_database.rst +++ b/automata/docs/core/database/json_vector_database.rst @@ -15,7 +15,7 @@ Related Symbols - ``VectorDatabaseProvider`` - ``Symbol`` - ``SymbolEmbedding`` -- ``automata.core.symbol_embedding.base.SymbolEmbedding`` +- ``automata.core.base.symbol_embedding.SymbolEmbedding`` Example ------- @@ -27,7 +27,7 @@ file. .. code:: python from automata.core.database.vector import JSONEmbeddingVectorDatabase - from automata.core.symbol.base import Symbol, SymbolEmbedding + from automata.core.base.symbol import Symbol, SymbolEmbedding # Create an instance of JSONEmbeddingVectorDatabase with a JSON file path file_path = "path/to/json_file.json" diff --git a/automata/docs/core/database/symbol_database_provider.rst b/automata/docs/core/database/symbol_database_provider.rst index 64cd7431..0cdb215b 100644 --- a/automata/docs/core/database/symbol_database_provider.rst +++ b/automata/docs/core/database/symbol_database_provider.rst @@ -12,7 +12,7 @@ Related Symbols --------------- - ``automata.tests.unit.test_symbol_search_tool.test_retrieve_source_code_by_symbol`` -- ``automata.core.symbol.base.Symbol`` +- ``automata.core.base.symbol.Symbol`` - ``automata.tests.unit.test_symbol_search_tool.test_symbol_references`` - ``automata.tests.unit.test_symbol_embedding.test_get_embedding`` - ``automata.core.memory_store.symbol_code_embedding.SymbolCodeEmbeddingHandler`` @@ -21,7 +21,7 @@ Related Symbols - ``automata.tests.unit.test_symbol_search.test_process_queries`` - ``automata.core.database.vector.VectorDatabaseProvider`` - ``automata.tests.unit.test_database_vector.test_lookup_symbol`` -- ``automata.core.symbol_embedding.base.SymbolEmbedding`` +- ``automata.core.base.symbol_embedding.SymbolEmbedding`` Example ------- @@ -34,7 +34,7 @@ implementing the required methods: import abc from typing import Any - from automata.core.symbol.base import Symbol, SymbolEmbedding + from automata.core.base.symbol import Symbol, SymbolEmbedding from automata.core.database.provider import SymbolDatabaseProvider class CustomSymbolDatabaseProvider(SymbolDatabaseProvider): diff --git a/automata/docs/core/database/vector_database_provider.rst b/automata/docs/core/database/vector_database_provider.rst index a2866d7e..b6fa5123 100644 --- a/automata/docs/core/database/vector_database_provider.rst +++ b/automata/docs/core/database/vector_database_provider.rst @@ -21,8 +21,8 @@ Related Symbols - ``automata.core.database.provider.SymbolDatabaseProvider`` - ``automata.core.database.vector.JSONEmbeddingVectorDatabase`` -- ``automata.core.symbol.base.Symbol`` -- ``automata.core.symbol_embedding.base.SymbolEmbedding`` +- ``automata.core.base.symbol.Symbol`` +- ``automata.core.base.symbol_embedding.SymbolEmbedding`` - ``automata.core.memory_store.embedding_types.SymbolEmbeddingHandler`` - ``automata.core.memory_store.symbol_code_embedding.SymbolCodeEmbeddingHandler`` @@ -37,7 +37,7 @@ base class. from automata.core.database.vector import VectorDatabaseProvider from typing import Dict, List - from automata.core.symbol.base import Symbol, SymbolEmbedding + from automata.core.base.symbol import Symbol, SymbolEmbedding class CustomVectorDatabaseProvider(VectorDatabaseProvider): def calculate_similarity(self, embedding: SymbolEmbedding) -> Dict[Symbol, float]: diff --git a/automata/docs/core/embedding/embedding_similarity.rst b/automata/docs/core/embedding/embedding_similarity.rst index c1019625..878c5266 100644 --- a/automata/docs/core/embedding/embedding_similarity.rst +++ b/automata/docs/core/embedding/embedding_similarity.rst @@ -18,7 +18,7 @@ Related Symbols - ``automata.tests.unit.test_symbol_embedding.test_update_embeddings`` - ``automata.core.database.vector.JSONEmbeddingVectorDatabase.calculate_similarity`` - ``automata.tests.unit.test_symbol_embedding.test_add_new_embedding`` -- ``automata.core.symbol_embedding.base.SymbolEmbedding`` +- ``automata.core.base.symbol_embedding.SymbolEmbedding`` Example ------- diff --git a/automata/docs/core/embedding/norm_type.rst b/automata/docs/core/embedding/norm_type.rst index a97d8004..89c8c484 100644 --- a/automata/docs/core/embedding/norm_type.rst +++ b/automata/docs/core/embedding/norm_type.rst @@ -31,7 +31,7 @@ Here’s an example demonstrating how to create an instance of from automata.core.symbol_embedding.similarity import SymbolSimilarity from automata.core.memory_store.embedding_types import NormType - from automata.core.symbol.base import SymbolEmbeddingHandler + from automata.core.base.symbol import SymbolEmbeddingHandler symbol_embedding_manager = SymbolEmbeddingHandler() norm_type = NormType.L1 diff --git a/automata/docs/core/embedding/open_ai_embedding.rst b/automata/docs/core/embedding/open_ai_embedding.rst index e56d663b..14f64d88 100644 --- a/automata/docs/core/embedding/open_ai_embedding.rst +++ b/automata/docs/core/embedding/open_ai_embedding.rst @@ -10,7 +10,7 @@ numpy array representing the symbol’s embedding. This class is closely related to ``automata.tests.conftest.mock_embedding``, ``automata.tests.unit.test_symbol_embedding.test_get_embedding``, and -``automata.core.symbol_embedding.base.SymbolEmbedding``. +``automata.core.base.symbol_embedding.SymbolEmbedding``. Overview -------- @@ -25,7 +25,7 @@ Related Symbols - ``automata.tests.conftest.mock_embedding`` - ``automata.tests.unit.test_symbol_embedding.test_get_embedding`` -- ``automata.core.symbol_embedding.base.SymbolEmbedding`` +- ``automata.core.base.symbol_embedding.SymbolEmbedding`` - ``automata.core.memory_store.embedding_types.EmbeddingProvider`` Usage Example diff --git a/automata/docs/core/embedding/symbol_code_embedding_builder.rst b/automata/docs/core/embedding/symbol_code_embedding_builder.rst index c91c4a47..59736af3 100644 --- a/automata/docs/core/embedding/symbol_code_embedding_builder.rst +++ b/automata/docs/core/embedding/symbol_code_embedding_builder.rst @@ -21,9 +21,9 @@ Related Symbols - ``automata.tests.unit.test_symbol_embedding.test_get_embedding`` - ``automata.tests.unit.test_database_vector.test_add_symbols`` -- ``automata.core.symbol_embedding.base.SymbolCodeEmbedding`` +- ``automata.core.base.symbol_embedding.SymbolCodeEmbedding`` - ``automata.core.memory_store.symbol_doc_embedding.SymbolDocEmbeddingBuilder`` -- ``automata.core.llm.embedding.SymbolEmbeddingBuilder`` +- ``automata.core.llm.core.SymbolEmbeddingBuilder`` - ``automata.core.memory_store.symbol_doc_embedding.SymbolDocEmbeddingHandler`` Example @@ -35,8 +35,8 @@ Below is an example demonstrating the use of the .. code:: python from automata.core.base.database.vector import VectorDatabaseProvider - from automata.core.llm.embedding import EmbeddingProvider, SymbolCodeEmbeddingBuilder - from automata.core.symbol.base import Symbol, SymbolCodeEmbedding + from automata.core.llm.core import EmbeddingProvider, SymbolCodeEmbeddingBuilder + from automata.core.base.symbol import Symbol, SymbolCodeEmbedding # create an instance of an embedding provider. embed_provider = EmbeddingProvider() diff --git a/automata/docs/core/embedding/symbol_code_embedding_handler.rst b/automata/docs/core/embedding/symbol_code_embedding_handler.rst index 86768464..956b1de6 100644 --- a/automata/docs/core/embedding/symbol_code_embedding_handler.rst +++ b/automata/docs/core/embedding/symbol_code_embedding_handler.rst @@ -24,9 +24,9 @@ Related Symbols --------------- - ``automata.core.base.database.vector.VectorDatabaseProvider`` -- ``automata.core.llm.embedding.EmbeddingProvider`` -- ``automata.core.symbol.base.Symbol`` -- ``automata.core.symbol_embedding.base.SymbolCodeEmbedding`` +- ``automata.core.llm.core.EmbeddingProvider`` +- ``automata.core.base.symbol.Symbol`` +- ``automata.core.base.symbol_embedding.SymbolCodeEmbedding`` - ``automata.core.symbol.symbol_utils.convert_to_fst_object`` Example @@ -38,8 +38,8 @@ The following example demonstrates how to create an instance of .. code:: python from automata.core.base.database.vector import VectorDatabaseProvider - from automata.core.llm.embedding import EmbeddingProvider, SymbolCodeEmbeddingHandler - from automata.core.symbol.base import Symbol + from automata.core.llm.core import EmbeddingProvider, SymbolCodeEmbeddingHandler + from automata.core.base.symbol import Symbol # Mocked objects for demonstration purposes mock_embedding_db = VectorDatabaseProvider() diff --git a/automata/docs/core/embedding/symbol_doc_embedding_builder.rst b/automata/docs/core/embedding/symbol_doc_embedding_builder.rst index 2f3dc63b..c3cf0490 100644 --- a/automata/docs/core/embedding/symbol_doc_embedding_builder.rst +++ b/automata/docs/core/embedding/symbol_doc_embedding_builder.rst @@ -18,13 +18,13 @@ Import Statements from automata.core.base.database.vector import VectorDatabaseProvider from automata.core.context.py.retriever import PyContextRetriever from automata.core.llm.completion import LLMChatCompletionProvider - from automata.core.llm.embedding import ( + from automata.core.llm.core import ( EmbeddingProvider, SymbolEmbeddingBuilder, SymbolEmbeddingHandler, ) - from automata.core.symbol.base import Symbol - from automata.core.symbol_embedding.base import SymbolDocEmbedding + from automata.core.base.symbol import Symbol + from automata.core.base.symbol_embedding import SymbolDocEmbedding from automata.core.symbol.search.symbol_search import SymbolSearch Example @@ -36,11 +36,11 @@ Here we will walk through an example of how to use the .. code:: python from automata.core.memory_store.symbol_doc_embedding import SymbolDocEmbeddingBuilder - from automata.core.llm.embedding import EmbeddingProvider + from automata.core.llm.core import EmbeddingProvider from automata.core.llm.completion import LLMChatCompletionProvider from automata.core.context.py.retriever import PyContextRetriever from automata.core.symbol.search.symbol_search import SymbolSearch - from automata.core.symbol.base import Symbol + from automata.core.base.symbol import Symbol # create instances of the dependent classes embedding_provider = EmbeddingProvider() diff --git a/automata/docs/core/embedding/symbol_doc_embedding_handler.rst b/automata/docs/core/embedding/symbol_doc_embedding_handler.rst index 811f13a7..011c126c 100644 --- a/automata/docs/core/embedding/symbol_doc_embedding_handler.rst +++ b/automata/docs/core/embedding/symbol_doc_embedding_handler.rst @@ -30,10 +30,10 @@ Related Symbols --------------- - ``automata.core.base.database.vector.VectorDatabaseProvider`` -- ``automata.core.llm.embedding.EmbeddingProvider`` -- ``automata.core.llm.embedding.SymbolEmbeddingHandler`` -- ``automata.core.symbol.base.Symbol`` -- ``automata.core.symbol_embedding.base.SymbolDocEmbedding`` +- ``automata.core.llm.core.EmbeddingProvider`` +- ``automata.core.llm.core.SymbolEmbeddingHandler`` +- ``automata.core.base.symbol.Symbol`` +- ``automata.core.base.symbol_embedding.SymbolDocEmbedding`` Example ------- diff --git a/automata/docs/core/embedding/symbol_embedding_handler.rst b/automata/docs/core/embedding/symbol_embedding_handler.rst index fdb5ce78..2561c00d 100644 --- a/automata/docs/core/embedding/symbol_embedding_handler.rst +++ b/automata/docs/core/embedding/symbol_embedding_handler.rst @@ -21,7 +21,7 @@ Related Symbols - ``automata.core.memory_store.symbol_code_embedding.SymbolCodeEmbeddingHandler`` - ``automata.core.memory_store.symbol_doc_embedding.SymbolDocEmbeddingHandler`` - ``automata.core.database.vector.VectorDatabaseProvider`` -- ``automata.core.symbol.base.Symbol`` +- ``automata.core.base.symbol.Symbol`` Example ------- @@ -34,7 +34,7 @@ The following code snippet demonstrates the usage of from automata.core.database.vector import JSONEmbeddingVectorDatabase from automata.core.memory_store.embedding_types import EmbeddingProvider from automata.core.memory_store.symbol_code_embedding import SymbolCodeEmbeddingHandler - from automata.core.symbol.base import Symbol + from automata.core.base.symbol import Symbol # Define the embedding database and embedding provider embedding_db = JSONEmbeddingVectorDatabase(database_filepath) diff --git a/automata/docs/core/embedding/symbol_similarity.rst b/automata/docs/core/embedding/symbol_similarity.rst index 8be1bc5b..a33741f2 100644 --- a/automata/docs/core/embedding/symbol_similarity.rst +++ b/automata/docs/core/embedding/symbol_similarity.rst @@ -23,10 +23,10 @@ similarity scores for all symbols in the database, respectively. Related Symbols --------------- -- ``automata.core.llm.embedding.SymbolEmbeddingHandler`` -- ``automata.core.llm.embedding.EmbeddingProvider`` -- ``automata.core.llm.embedding.EmbeddingNormType`` -- ``automata.core.symbol.base.Symbol`` +- ``automata.core.llm.core.SymbolEmbeddingHandler`` +- ``automata.core.llm.core.EmbeddingProvider`` +- ``automata.core.llm.core.EmbeddingNormType`` +- ``automata.core.base.symbol.Symbol`` Example ------- @@ -38,13 +38,13 @@ given query text. .. code:: python import numpy as np - from automata.core.llm.embedding import ( + from automata.core.llm.core import ( SymbolEmbeddingHandler, EmbeddingProvider, EmbeddingNormType, SymbolSimilarity, ) - from automata.core.symbol.base import Symbol + from automata.core.base.symbol import Symbol # Create a mock SymbolEmbeddingHandler and EmbeddingProvider symbol_embedding_handler = SymbolEmbeddingHandler(...) diff --git a/automata/docs/core/embedding/symbol_similarity_calculator.rst b/automata/docs/core/embedding/symbol_similarity_calculator.rst index 6d4b00c8..9e3fae7c 100644 --- a/automata/docs/core/embedding/symbol_similarity_calculator.rst +++ b/automata/docs/core/embedding/symbol_similarity_calculator.rst @@ -26,23 +26,23 @@ Related Symbols - ``automata.tests.unit.test_symbol_search_tool.test_exact_search`` - ``automata.core.agent.tool.tool_utils.DependencyFactory.create_symbol_doc_similarity`` - ``automata.tests.unit.test_symbol_search_tool.test_retrieve_source_code_by_symbol`` -- ``automata.core.symbol.base.Symbol`` +- ``automata.core.base.symbol.Symbol`` - ``automata.tests.conftest.symbol_search`` -- ``automata.core.llm.embedding.EmbeddingSimilarityCalculator`` +- ``automata.core.llm.core.EmbeddingSimilarityCalculator`` Usage Example ------------- .. code:: python - from automata.core.llm.embedding import ( + from automata.core.llm.core import ( EmbeddingNormType, EmbeddingProvider, SymbolEmbeddingHandler, ) from typing import Dict, List, Optional, Set from ann_embed.synthetic_provider import SyntheticProvider - from automata.core.symbol.base import Symbol + from automata.core.base.symbol import Symbol from numpy import random from automata.core.symbol_embedding.similarity import SymbolSimilarityCalculator diff --git a/automata/docs/core/llm/embedding_norm_type.rst b/automata/docs/core/llm/embedding_norm_type.rst index 1b3716c4..7329698b 100644 --- a/automata/docs/core/llm/embedding_norm_type.rst +++ b/automata/docs/core/llm/embedding_norm_type.rst @@ -13,9 +13,9 @@ Related Symbols - ``automata.tests.unit.test_symbol_embedding.test_add_new_embedding`` - ``automata.core.symbol_embedding.similarity.SymbolSimilarity`` - ``automata.tests.conftest.mock_embedding`` -- ``automata.core.symbol_embedding.base.SymbolEmbedding`` +- ``automata.core.base.symbol_embedding.SymbolEmbedding`` - ``automata.tests.unit.test_symbol_similarity.test_get_nearest_symbols_for_query`` -- ``automata.core.symbol_embedding.base.SymbolCodeEmbedding`` +- ``automata.core.base.symbol_embedding.SymbolCodeEmbedding`` - ``automata.tests.unit.test_symbol_embedding.test_update_embedding`` Example @@ -26,7 +26,7 @@ The following example demonstrates how to use ``EmbeddingNormType`` with .. code:: python - from automata.core.symbol.base import SymbolEmbedding + from automata.core.base.symbol import SymbolEmbedding from automata.core.symbol_embedding.similarity import SymbolSimilarity # Create an instance of SymbolSimilarity class with L2 normalization diff --git a/automata/docs/core/llm/embedding_provider.rst b/automata/docs/core/llm/embedding_provider.rst index 8d7cb584..dd25cc89 100644 --- a/automata/docs/core/llm/embedding_provider.rst +++ b/automata/docs/core/llm/embedding_provider.rst @@ -23,7 +23,7 @@ to provide embeddings for symbols using the ``numpy`` library: .. code:: python import numpy as np - from automata.core.llm.embedding import EmbeddingProvider + from automata.core.llm.core import EmbeddingProvider class CustomEmbeddingProvider(EmbeddingProvider): def build_embedding(self, symbol_source: str) -> np.ndarray: diff --git a/automata/docs/core/llm/embedding_similarity.rst b/automata/docs/core/llm/embedding_similarity.rst index 2f39b471..2ec525d7 100644 --- a/automata/docs/core/llm/embedding_similarity.rst +++ b/automata/docs/core/llm/embedding_similarity.rst @@ -12,7 +12,7 @@ Related Symbols --------------- - ``automata.core.base.database.vector.VectorDatabaseProvider`` -- ``automata.core.symbol.base.Symbol`` +- ``automata.core.base.symbol.Symbol`` - ``automata.core.symbol_embedding.similarity.SymbolSimilarity`` - ``automata.tests.unit.test_symbol_similarity.test_get_nearest_symbols_for_query`` @@ -36,7 +36,7 @@ for both methods: .. code:: python - from automata.core.llm.embedding import EmbeddingSimilarity + from automata.core.llm.core import EmbeddingSimilarity class DummySimilarity(EmbeddingSimilarity): def get_nearest_entries_for_query(self, query_text: str, k_nearest: int) -> Dict[Symbol, float]: diff --git a/automata/docs/core/llm/embedding_similarity_calculator.rst b/automata/docs/core/llm/embedding_similarity_calculator.rst index f2512b80..200a9034 100644 --- a/automata/docs/core/llm/embedding_similarity_calculator.rst +++ b/automata/docs/core/llm/embedding_similarity_calculator.rst @@ -19,7 +19,7 @@ and the symbols. Related Symbols --------------- -- ``automata.core.symbol.base.Symbol`` +- ``automata.core.base.symbol.Symbol`` - ``automata.tests.unit.test_symbol_similarity.test_get_nearest_symbols_for_query`` - ``automata.core.symbol_embedding.similarity.SymbolSimilarityCalculator`` - ``automata.core.agent.tool.tool_utils.DependencyFactory.create_symbol_code_similarity`` @@ -32,9 +32,9 @@ EmbeddingSimilarityCalculator - .. code:: python - from automata.core.llm.embedding import EmbeddingSimilarityCalculator + from automata.core.llm.core import EmbeddingSimilarityCalculator from typing import Dict - from automata.core.symbol.base import Symbol + from automata.core.base.symbol import Symbol class MyEmbeddingSimilarityCalculator(EmbeddingSimilarityCalculator): def calculate_query_similarity_dict(self, query_text: str) -> Dict[Symbol, float]: diff --git a/automata/docs/core/llm/providers/open_ai_embedding_provider.rst b/automata/docs/core/llm/providers/open_ai_embedding_provider.rst index 0aec54e4..3833073c 100644 --- a/automata/docs/core/llm/providers/open_ai_embedding_provider.rst +++ b/automata/docs/core/llm/providers/open_ai_embedding_provider.rst @@ -18,14 +18,14 @@ embedding. Related Symbols --------------- -- ``automata.core.llm.embedding.EmbeddingProvider`` +- ``automata.core.llm.core.EmbeddingProvider`` - ``automata.tests.unit.test_symbol_embedding.test_get_embedding`` - ``automata.tests.unit.test_symbol_embedding.test_get_embedding_exception`` - ``automata.tests.unit.test_symbol_embedding.test_add_new_embedding`` - ``automata.tests.unit.test_symbol_embedding.test_update_embedding`` - ``automata.core.memory_store.symbol_code_embedding.SymbolCodeEmbeddingHandler`` - ``automata.core.memory_store.symbol_doc_embedding.SymbolDocEmbeddingHandler`` -- ``automata.core.llm.embedding.SymbolEmbeddingHandler.__init__`` +- ``automata.core.llm.core.SymbolEmbeddingHandler.__init__`` Example ------- diff --git a/automata/docs/core/llm/symbol_embedding_builder.rst b/automata/docs/core/llm/symbol_embedding_builder.rst index 5f6a81ee..081e2511 100644 --- a/automata/docs/core/llm/symbol_embedding_builder.rst +++ b/automata/docs/core/llm/symbol_embedding_builder.rst @@ -30,7 +30,7 @@ Related Symbols - ``automata.core.memory_store.symbol_doc_embedding.SymbolDocEmbeddingBuilder`` - ``automata.tests.unit.test_symbol_embedding.test_update_embedding`` - ``automata.tests.unit.test_symbol_embedding.test_update_embeddings`` -- ``automata.core.symbol_embedding.base.SymbolCodeEmbedding`` +- ``automata.core.base.symbol_embedding.SymbolCodeEmbedding`` - ``automata.tests.unit.test_symbol_search_tool.symbol_search_tool_builder`` - ``automata.core.memory_store.symbol_code_embedding.SymbolCodeEmbeddingBuilder.build`` @@ -44,7 +44,7 @@ concrete class (``SymbolCodeEmbeddingBuilder``) that extends .. code:: python from automata.core.memory_store.symbol_code_embedding import SymbolCodeEmbeddingBuilder - from automata.core.llm.embedding import EmbeddingProvider + from automata.core.llm.core import EmbeddingProvider # Create an instance of EmbeddingProvider embedding_provider = EmbeddingProvider() diff --git a/automata/docs/core/llm/symbol_embedding_handler.rst b/automata/docs/core/llm/symbol_embedding_handler.rst index fbc29355..2e048dd5 100644 --- a/automata/docs/core/llm/symbol_embedding_handler.rst +++ b/automata/docs/core/llm/symbol_embedding_handler.rst @@ -23,7 +23,7 @@ Related Symbols - ``automata.core.memory_store.symbol_code_embedding.SymbolCodeEmbeddingHandler`` - ``automata.core.memory_store.symbol_doc_embedding.SymbolDocEmbeddingHandler`` -- ``automata.core.symbol.base.Symbol`` +- ``automata.core.base.symbol.Symbol`` - ``automata.core.base.database.vector.VectorDatabaseProvider`` Example @@ -36,7 +36,7 @@ specific symbol. .. code:: python from automata.core.memory_store.symbol_code_embedding import SymbolCodeEmbeddingHandler - from automata.core.symbol.base import Symbol + from automata.core.base.symbol import Symbol from automata.core.base.database.vector import JSONEmbeddingVectorDatabase from automata.core.memory_store.embedding_provider import EmbeddingProvider diff --git a/automata/docs/core/root_dict.rst b/automata/docs/core/root_dict.rst index 3d921b3c..fb2fd96d 100644 --- a/automata/docs/core/root_dict.rst +++ b/automata/docs/core/root_dict.rst @@ -17,7 +17,7 @@ Related Symbols - ``automata.core.utils.HandlerDict`` - ``automata.core.coding.py.module_loader.PyModuleLoader`` - ``automata.tests.unit.test_task_environment.TestURL`` -- ``automata.core.symbol.base.Symbol`` +- ``automata.core.base.symbol.Symbol`` Example ------- diff --git a/automata/docs/core/symbol/base/py_kind.rst b/automata/docs/core/symbol/base/py_kind.rst index 1cd61c2f..6fd2142f 100644 --- a/automata/docs/core/symbol/base/py_kind.rst +++ b/automata/docs/core/symbol/base/py_kind.rst @@ -13,7 +13,7 @@ Overview -------- ``SymbolDescriptor`` is primarily used in the context of -``automata.core.symbol.base.Symbol``, a class that represents a +``automata.core.base.symbol.Symbol``, a class that represents a standardized string representation for various programming elements such as packages, classes, methods, and more. By utilizing ``SymbolDescriptor``, it is possible to parse and store specific @@ -23,10 +23,10 @@ retrieval, or reference. Related Symbols --------------- -- ``automata.core.symbol.base.Symbol`` +- ``automata.core.base.symbol.Symbol`` - ``automata.core.symbol.parser._SymbolParser`` -- ``automata.core.symbol.base.SymbolDescriptor.PyKind`` -- ``automata.core.symbol.scip_pb2.Descriptor`` +- ``automata.core.base.symbol.SymbolDescriptor.PyKind`` +- ``automata.core.base.scip_pb2.Descriptor`` Examples -------- @@ -35,8 +35,8 @@ Below are some examples using the ``SymbolDescriptor`` class: .. code:: python - from automata.core.symbol.base import SymbolDescriptor - from automata.core.symbol.scip_pb2 import Descriptor as DescriptorProto + from automata.core.base.symbol import SymbolDescriptor + from automata.core.base.scip_pb2 import Descriptor as DescriptorProto # Create a SymbolDescriptor object for a Python method descriptor = SymbolDescriptor( diff --git a/automata/docs/core/symbol/graph/sub_graph.rst b/automata/docs/core/symbol/graph/sub_graph.rst index ac8d92aa..f390cc6c 100644 --- a/automata/docs/core/symbol/graph/sub_graph.rst +++ b/automata/docs/core/symbol/graph/sub_graph.rst @@ -19,8 +19,8 @@ Related Symbols --------------- - ``automata.core.symbol.graph.SymbolGraph`` -- ``automata.core.symbol.base.Symbol`` -- ``automata.core.symbol.scip_pb2.Index`` +- ``automata.core.base.symbol.Symbol`` +- ``automata.core.base.scip_pb2.Index`` - ``automata.core.symbol.symbol_utils.get_rankable_symbols`` - ``automata.tests.unit.test_symbol_rank.test_get_ranks_small_graph`` diff --git a/automata/docs/core/symbol/graph_builder.rst b/automata/docs/core/symbol/graph_builder.rst index 242ba9a7..dddd1dd9 100644 --- a/automata/docs/core/symbol/graph_builder.rst +++ b/automata/docs/core/symbol/graph_builder.rst @@ -22,7 +22,7 @@ Related Symbols - ``automata.core.symbol.graph._RelationshipManager`` - ``automata.core.symbol.graph._OccurrenceManager`` - ``automata.core.symbol.graph._CallerCalleeManager`` -- ``automata.core.symbol.base.Symbol`` +- ``automata.core.base.symbol.Symbol`` - ``automata.core.symbol.graph.SymbolGraph`` Example @@ -34,7 +34,7 @@ build a graph from a given index. .. code:: python from automata.core.symbol.graph import GraphBuilder - from automata.core.symbol.scip_pb2 import Index + from automata.core.base.scip_pb2 import Index # Assuming a valid index object index = Index() diff --git a/automata/docs/core/symbol/graph_processor.rst b/automata/docs/core/symbol/graph_processor.rst index 9a07ddad..8be6eb98 100644 --- a/automata/docs/core/symbol/graph_processor.rst +++ b/automata/docs/core/symbol/graph_processor.rst @@ -26,8 +26,8 @@ Import Statements: from automata.config import MAX_WORKERS from automata.core.coding.py.module_loader import py_module_loader from automata.core.symbol.parser import parse_symbol - from automata.core.symbol.scip_pb2 import Index, SymbolRole - from automata.core.symbol.base import ( + from automata.core.base.scip_pb2 import Index, SymbolRole + from automata.core.base.symbol import ( Symbol, SymbolDescriptor, SymbolFile, @@ -69,7 +69,7 @@ Builds a ``SymbolGraph`` from a corresponding Index. This class is responsible for constructing the SymbolGraph object used by the ``GraphProcessor``. -``automata.core.symbol.base.Symbol`` +``automata.core.base.symbol.Symbol`` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ A class which contains associated logic for a Symbol. This class diff --git a/automata/docs/core/symbol/search/symbol_rank.rst b/automata/docs/core/symbol/search/symbol_rank.rst index eacbe323..199e42a0 100644 --- a/automata/docs/core/symbol/search/symbol_rank.rst +++ b/automata/docs/core/symbol/search/symbol_rank.rst @@ -23,7 +23,7 @@ Related Symbols - ``networkx.DiGraph`` - ``automata.core.symbol.search.rank.SymbolRankConfig`` -- ``automata.core.symbol.base.Symbol`` +- ``automata.core.base.symbol.Symbol`` - ``automata.core.symbol.search.symbol_parser.parse_symbol`` Example @@ -37,7 +37,7 @@ importance of symbols in a directed graph. import networkx as nx from automata.core.symbol.search.rank import SymbolRank from automata.core.symbol.search.rank import SymbolRankConfig - from automata.core.symbol.base import Symbol + from automata.core.base.symbol import Symbol # create a directed graph G = nx.DiGraph() diff --git a/automata/docs/core/symbol/search/symbol_search.rst b/automata/docs/core/symbol/search/symbol_search.rst index 08c87d24..f41b5f7e 100644 --- a/automata/docs/core/symbol/search/symbol_search.rst +++ b/automata/docs/core/symbol/search/symbol_search.rst @@ -23,7 +23,7 @@ Related Symbols - ``automata.core.symbol_embedding.similarity.SymbolSimilarity`` - ``automata.core.symbol.search.rank.SymbolRank`` - ``automata.core.symbol.search.rank.SymbolRankConfig`` -- ``automata.core.symbol.base.Symbol`` +- ``automata.core.base.symbol.Symbol`` Example ------- diff --git a/automata/docs/core/symbol/symbol_code_embedding.rst b/automata/docs/core/symbol/symbol_code_embedding.rst index 21082ade..73a66d3a 100644 --- a/automata/docs/core/symbol/symbol_code_embedding.rst +++ b/automata/docs/core/symbol/symbol_code_embedding.rst @@ -18,9 +18,9 @@ and accessible way to store and manage symbol code embeddings. Related Symbols --------------- -- ``automata.core.symbol.base.Symbol`` -- ``automata.core.symbol_embedding.base.SymbolEmbedding`` -- ``automata.core.symbol_embedding.base.SymbolDocEmbedding`` +- ``automata.core.base.symbol.Symbol`` +- ``automata.core.base.symbol_embedding.SymbolEmbedding`` +- ``automata.core.base.symbol_embedding.SymbolDocEmbedding`` - ``automata.core.memory_store.symbol_code_embedding.SymbolCodeEmbeddingHandler`` Usage Example @@ -33,7 +33,7 @@ embedded vector. .. code:: python import numpy as np - from automata.core.symbol.base import Symbol, SymbolCodeEmbedding + from automata.core.base.symbol import Symbol, SymbolCodeEmbedding symbol = Symbol.from_string("some_symbol") # The desired Symbol object source_code = "def function_example():\n pass" @@ -43,7 +43,7 @@ embedded vector. Note that this example assumes that the ``Symbol`` object already exists. For more information about working with ``Symbol`` objects, -consult the ``automata.core.symbol.base.Symbol`` class documentation. +consult the ``automata.core.base.symbol.Symbol`` class documentation. Limitations ----------- diff --git a/automata/docs/core/symbol/symbol_descriptor.rst b/automata/docs/core/symbol/symbol_descriptor.rst index 5eff03be..ae3ce100 100644 --- a/automata/docs/core/symbol/symbol_descriptor.rst +++ b/automata/docs/core/symbol/symbol_descriptor.rst @@ -18,7 +18,7 @@ descriptor. Related Symbols --------------- -- ``automata.core.symbol.base.Symbol`` +- ``automata.core.base.symbol.Symbol`` - ``automata.core.symbol.parser.parse_symbol`` - ``automata.tests.unit.test_symbol_search_tool.test_retrieve_source_code_by_symbol`` @@ -30,8 +30,8 @@ The following example demonstrates how to create an instance of .. code:: python - from automata.core.symbol.base import SymbolDescriptor - from automata.core.symbol.scip_pb2 import Descriptor as DescriptorProto + from automata.core.base.symbol import SymbolDescriptor + from automata.core.base.scip_pb2 import Descriptor as DescriptorProto name = "automata.core.agent.agent_enums.ActionIndicator" suffix = DescriptorProto.Type diff --git a/automata/docs/core/symbol/symbol_doc_embedding.rst b/automata/docs/core/symbol/symbol_doc_embedding.rst index 0812b08b..999fe40b 100644 --- a/automata/docs/core/symbol/symbol_doc_embedding.rst +++ b/automata/docs/core/symbol/symbol_doc_embedding.rst @@ -21,10 +21,10 @@ Related Symbols --------------- - ``automata.core.memory_store.symbol_doc_embedding.SymbolDocEmbeddingHandler`` -- ``automata.core.symbol.scip_pb2.Descriptor`` +- ``automata.core.base.scip_pb2.Descriptor`` - ``automata.core.symbol.parser.parse_symbol`` -- ``automata.core.symbol_embedding.base.SymbolCodeEmbedding`` -- ``automata.core.symbol_embedding.base.SymbolEmbedding`` +- ``automata.core.base.symbol_embedding.SymbolCodeEmbedding`` +- ``automata.core.base.symbol_embedding.SymbolEmbedding`` Example ------- @@ -36,7 +36,7 @@ The following example demonstrates how to create a import numpy as np from automata.core.symbol.parser import parse_symbol - from automata.core.symbol.base import SymbolDocEmbedding + from automata.core.base.symbol import SymbolDocEmbedding symbol_str = "scip-python python automata 75482692a6fe30c72db516201a6f47d9fb4af065 `automata.core.agent.agent_enums`/ActionIndicator#" symbol = parse_symbol(symbol_str) diff --git a/automata/docs/core/symbol/symbol_embedding.rst b/automata/docs/core/symbol/symbol_embedding.rst index 4da0968e..2609766d 100644 --- a/automata/docs/core/symbol/symbol_embedding.rst +++ b/automata/docs/core/symbol/symbol_embedding.rst @@ -23,8 +23,8 @@ such as ``SymbolCodeEmbedding`` and ``SymbolDocEmbedding``. Related Symbols --------------- -- ``automata.core.symbol_embedding.base.SymbolCodeEmbedding`` -- ``automata.core.symbol_embedding.base.SymbolDocEmbedding`` +- ``automata.core.base.symbol_embedding.SymbolCodeEmbedding`` +- ``automata.core.base.symbol_embedding.SymbolDocEmbedding`` - ``automata.core.memory_store.symbol_code_embedding.SymbolCodeEmbeddingHandler`` Usage Example @@ -36,7 +36,7 @@ as well as a precomputed embedding array. .. code:: python import numpy as np - from automata.core.symbol.base import Symbol, SymbolCodeEmbedding + from automata.core.base.symbol import Symbol, SymbolCodeEmbedding symbol = Symbol( scheme="scip-python", diff --git a/automata/docs/core/symbol/symbol_file.rst b/automata/docs/core/symbol/symbol_file.rst index 10ea354a..823fb3b7 100644 --- a/automata/docs/core/symbol/symbol_file.rst +++ b/automata/docs/core/symbol/symbol_file.rst @@ -11,10 +11,10 @@ Related Symbols --------------- - ``automata.tests.unit.test_database_vector.test_lookup_symbol`` -- ``automata.core.symbol.base.Symbol`` +- ``automata.core.base.symbol.Symbol`` - ``automata.tests.unit.test_symbol_graph.test_get_all_files`` - ``automata.tests.unit.test_symbol_search_tool.test_retrieve_source_code_by_symbol`` -- ``automata.core.symbol.base.SymbolReference`` +- ``automata.core.base.symbol.SymbolReference`` - ``automata.tests.unit.test_database_vector.test_add_symbol`` - ``automata.tests.unit.test_database_vector.test_add_symbols`` - ``automata.core.base.database.vector.JSONEmbeddingVectorDatabase`` @@ -29,7 +29,7 @@ The following example demonstrates how to create an instance of .. code:: python - from automata.core.symbol.base import SymbolFile + from automata.core.base.symbol import SymbolFile file_path = "path/to/symbol_file.txt" symbol_file = SymbolFile(file_path) diff --git a/automata/docs/core/symbol/symbol_graph.rst b/automata/docs/core/symbol/symbol_graph.rst index 6eda1267..56cad1c9 100644 --- a/automata/docs/core/symbol/symbol_graph.rst +++ b/automata/docs/core/symbol/symbol_graph.rst @@ -21,8 +21,8 @@ Related Symbols --------------- - ``automata.core.symbol.parser.parse_symbol`` -- ``automata.core.symbol.scip_pb2.Index`` -- ``automata.core.symbol.base.Symbol`` +- ``automata.core.base.scip_pb2.Index`` +- ``automata.core.base.symbol.Symbol`` - ``automata.core.symbol.graph.GraphBuilder`` - ``automata.core.symbol.graph._SymbolGraphNavigator`` - ``automata.core.agent.tool.tool_utils.DependencyFactory.create_symbol_graph`` diff --git a/automata/docs/core/symbol/symbol_package.rst b/automata/docs/core/symbol/symbol_package.rst index 16775406..6ff90070 100644 --- a/automata/docs/core/symbol/symbol_package.rst +++ b/automata/docs/core/symbol/symbol_package.rst @@ -8,9 +8,9 @@ from URI string format. Related Symbols --------------- -- ``automata.core.symbol.scip_pb2.Descriptor`` +- ``automata.core.base.scip_pb2.Descriptor`` - ``automata.core.symbol.parser.parse_symbol`` -- ``automata.core.symbol.base.Symbol`` +- ``automata.core.base.symbol.Symbol`` - ``automata.tests.unit.test_symbol_parser.test_parse_symbol`` Methods diff --git a/automata/docs/core/symbol/symbol_parser.rst b/automata/docs/core/symbol/symbol_parser.rst index 4facaf62..4382b1b2 100644 --- a/automata/docs/core/symbol/symbol_parser.rst +++ b/automata/docs/core/symbol/symbol_parser.rst @@ -16,10 +16,10 @@ descriptor suffixes. Related Symbols --------------- -- ``automata.core.symbol.base.Symbol`` +- ``automata.core.base.symbol.Symbol`` - ``automata.core.symbol.parser.parse_symbol`` - ``automata.core.symbol.graph.SymbolGraph`` -- ``automata.core.symbol.base.SymbolDescriptor`` +- ``automata.core.base.symbol.SymbolDescriptor`` - ``automata.core.memory_store.symbol_code_embedding.SymbolCodeEmbeddingHandler`` - ``automata.core.context.py_context.retriever.PyContextRetriever`` diff --git a/automata/docs/core/symbol/symbol_reference.rst b/automata/docs/core/symbol/symbol_reference.rst index 31b929b6..cf5b9164 100644 --- a/automata/docs/core/symbol/symbol_reference.rst +++ b/automata/docs/core/symbol/symbol_reference.rst @@ -19,19 +19,19 @@ comparison of symbol reference objects, even across different files. Related Symbols --------------- -- ``automata.core.symbol.base.Symbol`` +- ``automata.core.base.symbol.Symbol`` - ``automata.core.symbol.graph._SymbolGraphNavigator._get_symbol_references_in_scope`` - ``automata.tests.unit.test_symbol_search.tool.test_symbol_references`` - ``automata.tests.unit.test_symbol_search_tool.test_retrieve_source_code_by_symbol`` - ``automata.tests.unit.test_database_vector.test_lookup_symbol`` -- ``automata.core.symbol_embedding.base.SymbolDocEmbedding`` +- ``automata.core.base.symbol_embedding.SymbolDocEmbedding`` Example ------- .. code:: python - from automata.core.symbol.base import Symbol, SymbolReference + from automata.core.base.symbol import Symbol, SymbolReference # Create two Symbol objects symbol_1 = Symbol.from_string("scip-python python automata 0.1.0 AutomataNamespace/ClassName#") diff --git a/automata/docs/core/symbol/symbol_types/py_kind.rst b/automata/docs/core/symbol/symbol_types/py_kind.rst index 1cd61c2f..6fd2142f 100644 --- a/automata/docs/core/symbol/symbol_types/py_kind.rst +++ b/automata/docs/core/symbol/symbol_types/py_kind.rst @@ -13,7 +13,7 @@ Overview -------- ``SymbolDescriptor`` is primarily used in the context of -``automata.core.symbol.base.Symbol``, a class that represents a +``automata.core.base.symbol.Symbol``, a class that represents a standardized string representation for various programming elements such as packages, classes, methods, and more. By utilizing ``SymbolDescriptor``, it is possible to parse and store specific @@ -23,10 +23,10 @@ retrieval, or reference. Related Symbols --------------- -- ``automata.core.symbol.base.Symbol`` +- ``automata.core.base.symbol.Symbol`` - ``automata.core.symbol.parser._SymbolParser`` -- ``automata.core.symbol.base.SymbolDescriptor.PyKind`` -- ``automata.core.symbol.scip_pb2.Descriptor`` +- ``automata.core.base.symbol.SymbolDescriptor.PyKind`` +- ``automata.core.base.scip_pb2.Descriptor`` Examples -------- @@ -35,8 +35,8 @@ Below are some examples using the ``SymbolDescriptor`` class: .. code:: python - from automata.core.symbol.base import SymbolDescriptor - from automata.core.symbol.scip_pb2 import Descriptor as DescriptorProto + from automata.core.base.symbol import SymbolDescriptor + from automata.core.base.scip_pb2 import Descriptor as DescriptorProto # Create a SymbolDescriptor object for a Python method descriptor = SymbolDescriptor( diff --git a/automata/docs/tests/unit/test_url.rst b/automata/docs/tests/unit/test_url.rst index 5c6d34fc..0b8a005c 100644 --- a/automata/docs/tests/unit/test_url.rst +++ b/automata/docs/tests/unit/test_url.rst @@ -17,7 +17,7 @@ Related Symbols - ``automata.tests.unit.test_tool.test_tool_run`` - ``automata.core.agent.task.task.AutomataTask`` - ``automata.tests.unit.test_tool.test_tool`` -- ``automata.core.symbol.base.Symbol`` +- ``automata.core.base.symbol.Symbol`` - ``automata.tests.unit.test_tool.TestTool`` - ``automata.core.coding.py.reader.PyReader`` - ``automata.tests.unit.test_symbol_search_tool.test_retrieve_source_code_by_symbol`` diff --git a/automata/tests/regression/test_symbol_searcher_regression.py b/automata/tests/regression/test_symbol_searcher_regression.py index 2242d4b2..d0f8b36e 100644 --- a/automata/tests/regression/test_symbol_searcher_regression.py +++ b/automata/tests/regression/test_symbol_searcher_regression.py @@ -53,7 +53,7 @@ def check_hits(expected_in_top_hits, found_top_hits): "run_agent", ], "database": [ - "JSONEmbeddingVectorDatabase", + "JSONSymbolEmbeddingVectorDatabase", "AgentConversationDatabase", "SymbolDatabaseProvider", "VectorDatabaseProvider", diff --git a/automata/tests/unit/test_automata_agent_builder.py b/automata/tests/unit/test_automata_agent_builder.py index 58a5341c..45a769c4 100644 --- a/automata/tests/unit/test_automata_agent_builder.py +++ b/automata/tests/unit/test_automata_agent_builder.py @@ -12,7 +12,7 @@ def test_automata_agent_init(automata_agent): assert automata_agent.config.model == "gpt-4" assert automata_agent.config.session_id is not None assert len(automata_agent.config.tools) > 0 - assert len(automata_agent.agent_conversations) == 5 + assert len(automata_agent.agent_conversation_database) == 5 def test_builder_default_config(automata_agent_config_builder): diff --git a/automata/tests/unit/test_conversation_database.py b/automata/tests/unit/test_conversation_database.py index 6e695587..fa44a31f 100644 --- a/automata/tests/unit/test_conversation_database.py +++ b/automata/tests/unit/test_conversation_database.py @@ -3,7 +3,9 @@ import pytest from automata.core.llm.completion import LLMChatMessage -from automata.core.memory_store.agent_conversations import AgentConversationDatabase +from automata.core.memory_store.agent_conversation_database import ( + AgentConversationDatabase, +) @pytest.fixture(scope="module", autouse=True) diff --git a/automata/tests/unit/test_database_vector.py b/automata/tests/unit/test_database_vector.py index 23eacce7..fed3cc67 100644 --- a/automata/tests/unit/test_database_vector.py +++ b/automata/tests/unit/test_database_vector.py @@ -1,22 +1,24 @@ import pytest -from automata.core.base.database.vector import JSONEmbeddingVectorDatabase -from automata.core.symbol_embedding.base import SymbolCodeEmbedding +from automata.core.base.symbol_embedding import ( + JSONSymbolEmbeddingVectorDatabase, + SymbolCodeEmbedding, +) def test_init_vector(temp_output_filename): - JSONEmbeddingVectorDatabase(temp_output_filename) + JSONSymbolEmbeddingVectorDatabase(temp_output_filename) def test_add_symbol(temp_output_filename, symbols): - vector_db = JSONEmbeddingVectorDatabase(temp_output_filename) + vector_db = JSONSymbolEmbeddingVectorDatabase(temp_output_filename) symbol = symbols[0] embedded_symbol = SymbolCodeEmbedding(symbol, "x", [1, 2, 3]) vector_db.add(embedded_symbol) def test_delete_symbol(temp_output_filename, symbols): - vector_db = JSONEmbeddingVectorDatabase(temp_output_filename) + vector_db = JSONSymbolEmbeddingVectorDatabase(temp_output_filename) symbol = symbols[0] embedded_symbol = SymbolCodeEmbedding(symbol, "x", [1, 2, 3]) vector_db.add(embedded_symbol) @@ -24,7 +26,7 @@ def test_delete_symbol(temp_output_filename, symbols): def test_add_symbols(temp_output_filename, symbols): - vector_db = JSONEmbeddingVectorDatabase(temp_output_filename) + vector_db = JSONSymbolEmbeddingVectorDatabase(temp_output_filename) embedded_symbol_0 = SymbolCodeEmbedding(symbols[0], "x", [1, 2, 3]) vector_db.add(embedded_symbol_0) embedded_symbol_1 = SymbolCodeEmbedding(symbols[1], "y", [1, 2, 3, 4]) @@ -32,7 +34,7 @@ def test_add_symbols(temp_output_filename, symbols): def test_lookup_symbol(temp_output_filename, symbols): - vector_db = JSONEmbeddingVectorDatabase(temp_output_filename) + vector_db = JSONSymbolEmbeddingVectorDatabase(temp_output_filename) embedded_symbol_0 = SymbolCodeEmbedding(symbols[0], "x", [1, 2, 3]) vector_db.add(embedded_symbol_0) embedded_symbol_1 = SymbolCodeEmbedding(symbols[1], "y", [1, 2, 3, 4]) @@ -42,7 +44,7 @@ def test_lookup_symbol(temp_output_filename, symbols): def test_lookup_symbol_fail(temp_output_filename, symbols): - vector_db = JSONEmbeddingVectorDatabase(temp_output_filename) + vector_db = JSONSymbolEmbeddingVectorDatabase(temp_output_filename) embedded_symbol_0 = SymbolCodeEmbedding(symbols[0], "x", [1, 2, 3]) vector_db.add(embedded_symbol_0) embedded_symbol_1 = SymbolCodeEmbedding(symbols[1], "y", [1, 2, 3, 4]) @@ -55,7 +57,7 @@ def test_lookup_symbol_fail(temp_output_filename, symbols): def test_save(temp_output_filename, symbols): - vector_db = JSONEmbeddingVectorDatabase(temp_output_filename) + vector_db = JSONSymbolEmbeddingVectorDatabase(temp_output_filename) embedded_symbol_0 = SymbolCodeEmbedding(symbols[0], "x", [1, 2, 3]) vector_db.add(embedded_symbol_0) embedded_symbol_1 = SymbolCodeEmbedding(symbols[1], "y", [1, 2, 3, 4]) @@ -64,14 +66,14 @@ def test_save(temp_output_filename, symbols): def test_load(temp_output_filename, symbols): - vector_db = JSONEmbeddingVectorDatabase(temp_output_filename) + vector_db = JSONSymbolEmbeddingVectorDatabase(temp_output_filename) embedded_symbol_0 = SymbolCodeEmbedding(symbols[0], "x", [1, 2, 3]) vector_db.add(embedded_symbol_0) embedded_symbol_1 = SymbolCodeEmbedding(symbols[1], "y", [1, 2, 3, 4]) vector_db.add(embedded_symbol_1) vector_db.save() - vector_db_2 = JSONEmbeddingVectorDatabase(temp_output_filename) + vector_db_2 = JSONSymbolEmbeddingVectorDatabase(temp_output_filename) embedded_symbol_0 = vector_db_2.get(symbols[0].dotpath) embedded_symbol_1 = vector_db_2.get(symbols[1].dotpath) diff --git a/automata/tests/unit/test_symbol_embedding.py b/automata/tests/unit/test_symbol_embedding.py index 4ffcaf24..046b309f 100644 --- a/automata/tests/unit/test_symbol_embedding.py +++ b/automata/tests/unit/test_symbol_embedding.py @@ -2,10 +2,12 @@ import numpy as np -from automata.core.base.database.vector import JSONEmbeddingVectorDatabase -from automata.core.llm.embedding import SymbolEmbeddingBuilder +from automata.core.base.symbol_embedding import ( + JSONSymbolEmbeddingVectorDatabase, + SymbolCodeEmbedding, + SymbolEmbeddingBuilder, +) from automata.core.memory_store.symbol_code_embedding import SymbolCodeEmbeddingHandler -from automata.core.symbol_embedding.base import SymbolCodeEmbedding def test_update_embeddings( @@ -23,8 +25,8 @@ def test_update_embeddings( mock_provider = Mock(SymbolEmbeddingBuilder) # mock_provider.build_embedding_array.return_value = mock_embedding - # Mock JSONEmbeddingVectorDatabase methods - mock_db = MagicMock(JSONEmbeddingVectorDatabase) + # Mock JSONSymbolEmbeddingVectorDatabase methods + mock_db = MagicMock(JSONSymbolEmbeddingVectorDatabase) mock_db.get.return_value = SymbolCodeEmbedding( mock_simple_method_symbols[0], mock_embedding, "symbol_source" ) @@ -55,8 +57,8 @@ def test_get_embedding( mock_provider = Mock(SymbolEmbeddingBuilder) # mock_provider.build_embedding_array.return_value = mock_embedding - # Mock JSONEmbeddingVectorDatabase methods - mock_db = MagicMock(JSONEmbeddingVectorDatabase) + # Mock JSONSymbolEmbeddingVectorDatabase methods + mock_db = MagicMock(JSONSymbolEmbeddingVectorDatabase) mock_db.get.return_value = SymbolCodeEmbedding( mock_simple_method_symbols[0], "symbol_source", mock_embedding ) @@ -82,8 +84,8 @@ def test_add_new_embedding(monkeypatch, mock_simple_method_symbols): mock_provider = Mock(SymbolEmbeddingBuilder) # mock_provider.build_embedding_array.return_value = [1, 2, 3] - # Mock JSONEmbeddingVectorDatabase methods - mock_db = MagicMock(JSONEmbeddingVectorDatabase) + # Mock JSONSymbolEmbeddingVectorDatabase methods + mock_db = MagicMock(JSONSymbolEmbeddingVectorDatabase) mock_db.data = [] mock_db.contains = lambda x: False mock_db.add = lambda x: mock_db.data.append(x) @@ -106,8 +108,8 @@ def test_update_embedding(monkeypatch, mock_simple_method_symbols): # Mock EmbeddingProvider methods mock_provider = Mock(SymbolEmbeddingBuilder) - # Mock JSONEmbeddingVectorDatabase methods - mock_db = MagicMock(JSONEmbeddingVectorDatabase) + # Mock JSONSymbolEmbeddingVectorDatabase methods + mock_db = MagicMock(JSONSymbolEmbeddingVectorDatabase) mock_db.data = [] mock_db.contains = lambda x: False mock_db.add = lambda x: mock_db.data.append(x) @@ -147,8 +149,8 @@ def test_get_embedding_exception(monkeypatch, mock_simple_method_symbols): mock_provider = Mock(SymbolEmbeddingBuilder) mock_provider.build.side_effect = Exception("Test exception") - # Mock JSONEmbeddingVectorDatabase methods - mock_db = MagicMock(JSONEmbeddingVectorDatabase) + # Mock JSONSymbolEmbeddingVectorDatabase methods + mock_db = MagicMock(JSONSymbolEmbeddingVectorDatabase) mock_db.data = [] mock_db.contains = lambda x: False mock_db.add = lambda x: mock_db.data.append(x) diff --git a/automata/tests/unit/test_symbol_graph.py b/automata/tests/unit/test_symbol_graph.py index a6280654..1ba4d15a 100644 --- a/automata/tests/unit/test_symbol_graph.py +++ b/automata/tests/unit/test_symbol_graph.py @@ -1,4 +1,4 @@ -from automata.core.symbol.base import Symbol, SymbolFile +from automata.core.base.symbol import Symbol, SymbolFile from automata.tests.utils.factories import symbol_graph_static_test # noqa: F401 diff --git a/automata/tests/unit/test_symbol_similarity.py b/automata/tests/unit/test_symbol_similarity.py index 61ca4a20..61669603 100644 --- a/automata/tests/unit/test_symbol_similarity.py +++ b/automata/tests/unit/test_symbol_similarity.py @@ -2,10 +2,13 @@ import numpy as np -from automata.core.base.database.vector import JSONEmbeddingVectorDatabase -from automata.core.llm.embedding import EmbeddingProvider, SymbolEmbeddingBuilder +from automata.core.base.embedding import EmbeddingProvider +from automata.core.base.symbol_embedding import ( + JSONSymbolEmbeddingVectorDatabase, + SymbolCodeEmbedding, + SymbolEmbeddingBuilder, +) from automata.core.memory_store.symbol_code_embedding import SymbolCodeEmbeddingHandler -from automata.core.symbol_embedding.base import SymbolCodeEmbedding from automata.core.symbol_embedding.similarity import SymbolSimilarityCalculator @@ -27,8 +30,8 @@ def test_get_nearest_symbols_for_query( symbol=symbol3, vector=np.array([0, 0, 1, 0]), source_code="symbol3" ) - # Mock JSONEmbeddingVectorDatabase methods - embedding_db = JSONEmbeddingVectorDatabase(temp_output_filename) + # Mock JSONSymbolEmbeddingVectorDatabase methods + embedding_db = JSONSymbolEmbeddingVectorDatabase(temp_output_filename) embedding_db.add(embedding1) embedding_db.add(embedding2) embedding_db.add(embedding3) diff --git a/automata/tests/utils/factories.py b/automata/tests/utils/factories.py index 3daac1ba..035da49a 100644 --- a/automata/tests/utils/factories.py +++ b/automata/tests/utils/factories.py @@ -3,13 +3,13 @@ import pytest from automata.config.base import ConfigCategory -from automata.core.base.database.vector import JSONEmbeddingVectorDatabase +from automata.core.base.symbol_embedding import JSONSymbolEmbeddingVectorDatabase from automata.core.llm.providers.openai import OpenAIEmbeddingProvider from automata.core.memory_store.symbol_code_embedding import SymbolCodeEmbeddingHandler from automata.core.symbol.graph import SymbolGraph from automata.core.symbol.search.rank import SymbolRankConfig from automata.core.symbol.search.symbol_search import SymbolSearch -from automata.core.symbol_embedding.embedding_builders import SymbolCodeEmbeddingBuilder +from automata.core.symbol_embedding.builders import SymbolCodeEmbeddingBuilder from automata.core.symbol_embedding.similarity import SymbolSimilarityCalculator from automata.core.utils import get_config_fpath @@ -41,7 +41,7 @@ def symbol_search_live() -> SymbolSearch: code_embedding_fpath = os.path.join( get_config_fpath(), ConfigCategory.SYMBOL.value, "symbol_code_embedding.json" ) - code_embedding_db = JSONEmbeddingVectorDatabase(code_embedding_fpath) + code_embedding_db = JSONSymbolEmbeddingVectorDatabase(code_embedding_fpath) embedding_provider = OpenAIEmbeddingProvider() embedding_builder = SymbolCodeEmbeddingBuilder(embedding_provider) code_embedding_handler = SymbolCodeEmbeddingHandler(code_embedding_db, embedding_builder)