[docs] Added READMEs in each directory

README.md

@@ -48,12 +48,12 @@ If you want to look at implementations of other agents that others have develope
 python pyopenagi/agents/interact.py --mode download --agent <author_name/agent_name>
 ```
 
-
 ## 🚀 Contributions
+
 For detailed information on how to contribute, see [CONTRIBUTE](./CONTRIBUTE.md). If you would like to contribute to the codebase, [issues](https://github.com/agiresearch/OpenAGI/issues) or [pull requests](https://github.com/agiresearch/OpenAGI/pulls) are always welcome!
 
 ## 🖋️ Research
-Please check out our [implementation](./research) for our research paper [OpenAGI: When LLM Meets Domain Experts](https://arxiv.org/abs/2304.04370).
+Please check out our [implementation](https://github.com/agiresearch/OpenAGI/tree/research) for our research paper [OpenAGI: When LLM Meets Domain Experts](https://arxiv.org/abs/2304.04370).
 
 ```
 @article{openagi,

pyopenagi/README.md

@@ -0,0 +1,8 @@
+# pyopenagi
+
+The internal implementation for OpenAGI.
+
+1. `agents/` contains the agent implementation all future agents have to follow.
+2. `queues/` contains the class implementation for queues.
+3. `utils/` contains some helpful internal utilities.
+4. `tools/` contains the tools the agents can use.

pyopenagi/agents/README.md

@@ -0,0 +1,4 @@
+# pyopenagi/agents
+This folder contains the base implementation for running the agents, as well as the handlers for running multiple agents in Threads in `agent_process.py` and `agent_factory.py`
+
+In `example/` we have some example agents. You can add agents to that directory or to `your-cool-identifier/` to show your agent off in the main repo. However, it is recommended to use the agent database over submitting a pull request, unless it is for demo purposes.

pyopenagi/agents/agent_factory.py

@@ -1,3 +1,8 @@
+# this class runs the "AgentProcess" for the agents and holds the pool
+# of agents currently usable
+# this class isn't actually instantiated until the user instantiates
+# a specific agent like MathAgent
+
 import heapq
 from threading import Lock, Event
 from pympler import asizeof
@@ -6,18 +11,20 @@
 import importlib
 
 class AgentFactory:
-    def __init__(self,
-                 agent_process_queue,
-                 agent_process_factory,
-                 agent_log_mode
-        ):
+    """ duplicate of AgentProcessFactory """
+
+    def __init__(self, llm, agent_process_queue, agent_process_factory, agent_log_mode):
+        # 256 agent ids heapified similar to AgentProcessFactory
         self.max_aid = 256
         # self.llm = llm
         self.aid_pool = [i for i in range(self.max_aid)]
         heapq.heapify(self.aid_pool)
         self.agent_process_queue = agent_process_queue
+
         self.agent_process_factory = agent_process_factory
 
+
+        # added to with index aid
         self.current_agents = {}
 
         self.current_agents_lock = Lock()
@@ -59,6 +66,7 @@ def activate_agent(self, agent_name, task_input):
 
         agent_class = self.load_agent_instance(agent_name)
 
+        """ initialize each agent """
         agent = agent_class(
             agent_name = agent_name,
             task_input = task_input,
@@ -77,6 +85,8 @@ def activate_agent(self, agent_name, task_input):
         return agent
 
     def run_agent(self, agent_name, task_input):
+        """ run the Thread and return output """
+
         agent = self.activate_agent(
             agent_name=agent_name,
             task_input=task_input
@@ -87,6 +97,7 @@ def run_agent(self, agent_name, task_input):
         return output
 
     def print_agent(self):
+        """ print all agent information in a nice organized format """
         headers = ["Agent ID", "Agent Name", "Created Time", "Status", "Memory Usage"]
         data = []
         for id, agent in self.current_agents.items():
@@ -101,6 +112,7 @@ def print_agent(self):
 
 
     def print(self, headers, data):
+        """ generalized table printer for the agent data """
         # align output
         column_widths = [
             max(len(str(row[i])) for row in [headers] + data) for i in range(len(headers))
@@ -116,9 +128,12 @@ def print(self, headers, data):
 
 
     def format_row(self, row, widths, align="<"):
+        """ helper utility for print """
         row_str = " | ".join(f"{str(item):{align}{widths[i]}}" for i, item in enumerate(row))
         return row_str
 
     def deactivate_agent(self, aid):
+        """ remove the agent id from the pool and allow it to be re-used """
+
         self.current_agents.pop(aid)
         heapq.heappush(self.aid_pool, aid)

pyopenagi/agents/agent_process.py

@@ -1,10 +1,18 @@
+# defines a process holder for agents to use in a single class
+# used in the base implementation for agents
+# this class isn't actually instantiated until the user instantiates
+# a specific agent like MathAgent
+
 import heapq
 
 from threading import Thread, Lock, Event
 
 from ..utils.chat_template import Query
 
 class AgentProcess:
+    """
+    each agent holds the values defined in the constructor
+    """
     def __init__(self,
             agent_name: str,
             query: Query
@@ -25,6 +33,10 @@ def __init__(self,
         self.start_time = None
         self.end_time = None
 
+    #######################
+    # getters and setters #
+    #######################
+
     def set_created_time(self, time):
         self.created_time = time
 
@@ -78,7 +90,14 @@ class LLMRequestProcess(AgentProcess):
     pass
 
 class AgentProcessFactory:
+    """ 
+    used by AgentFactory
+    right now it is only set in agent activation
+    """
+
     def __init__(self, agent_process_log_mode = None):
+        # make a heap of every pid 1..1024 to remove from when agent created
+        # so pids can be reused
         self.max_pid = 1024
         self.pid_pool = [i for i in range(self.max_pid)]
         heapq.heapify(self.pid_pool)
@@ -94,8 +113,9 @@ def __init__(self, agent_process_log_mode = None):
         self.agent_process_log_mode = agent_process_log_mode
 
     def activate_agent_process(self, agent_name, query):
+        """ run each agent and lock it """
         if not self.terminate_signal.is_set():
-            with self.current_agent_processes_lock:
+           with self.current_agent_processes_lock:
                 agent_process = AgentProcess(
                     agent_name = agent_name,
                     query = query
@@ -107,6 +127,7 @@ def activate_agent_process(self, agent_name, query):
                 return agent_process
 
     def print_agent_process(self):
+        """ print agent data in a clean format """
         headers = ["Agent Process ID", "Agent Name", "Created Time", "Status"]
         data = []
         for id, agent_process in self.current_agent_processes.items():
@@ -121,6 +142,8 @@ def print_agent_process(self):
 
 
     def print(self, headers, data):
+        """ separate headers and data in printing """
+
         # align output
         column_widths = [
             max(len(str(row[i])) for row in [headers] + data) for i in range(len(headers))
@@ -136,10 +159,12 @@ def print(self, headers, data):
 
 
     def format_row(self, row, widths, align="<"):
+        """ helper for print """
         row_str = " | ".join(f"{str(item):{align}{widths[i]}}" for i, item in enumerate(row))
         return row_str
 
     def deactivate_agent_process(self, pid):
+        """ after agent stops reallow pid """
         self.current_agent_processes.pop(pid)
         heapq.heappush(self.pid_pool, pid)
 

pyopenagi/agents/base_agent.py

@@ -1,3 +1,6 @@
+# stub implementation for agents
+# defines a class standard each agent has to subclass
+
 import os
 
 import json
@@ -19,16 +22,24 @@
 from ..queues.llm_request_queue import LLMRequestQueue
 
 class CustomizedThread(Thread):
+    """ provides a specific agent runtime """
+
     def __init__(self, target, args=()):
         super().__init__()
         self.target = target
         self.args = args
         self.result = None
 
     def run(self):
+        """
+        instead of creating a Thread, it'll run a function and hold the value
+        in BaseAgent, it's only usage, it creates a Thread and sets certain 
+        values for the AgentProcess to provide runtime diagnostics
+        """
         self.result = self.target(*self.args)
 
     def join(self):
+        """ returns the result from the custom Thread runtime """
         super().join()
         return self.result
 
@@ -162,6 +173,10 @@ def setup_logger(self):
         return logger
 
     def load_config(self):
+        """  
+        loads each agent config with the values in the json
+        such as the system prompt, functions 
+        """
         script_path = os.path.abspath(__file__)
         script_dir = os.path.dirname(script_path)
         config_file = os.path.join(script_dir, self.agent_name, "config.json")
@@ -174,13 +189,16 @@ def get_response(self,
             query,
             temperature=0.0
         ):
+        """ value of the agent """
         thread = CustomizedThread(target=self.query_loop, args=(query, ))
         thread.start()
         return thread.join()
 
     def query_loop(self, query):
+        """ custom function for the CustomizedThread """
         agent_process = self.create_agent_request(query)
 
+        # because it might have to run multiple times
         completed_response, start_times, end_times, waiting_times, turnaround_times = "", [], [], [], []
 
         while agent_process.get_status() != "done":
@@ -200,6 +218,8 @@ def query_loop(self, query):
                     f"Suspended due to the reach of time limit ({agent_process.get_time_limit()}s). Current result is: {completed_response.response_message}\n",
                     level="suspending"
                 )
+
+            # data from the AgentProcess temporarily held
             start_time = agent_process.get_start_time()
             end_time = agent_process.get_end_time()
             waiting_time = start_time - agent_process.get_created_time()
@@ -216,6 +236,7 @@ def query_loop(self, query):
         return completed_response, start_times, end_times, waiting_times, turnaround_times
 
     def create_agent_request(self, query):
+        """ uses AgentFactory to initialize the agent in the pool """
         agent_process = self.agent_process_factory.activate_agent_process(
             agent_name = self.agent_name,
             query = query
@@ -238,6 +259,14 @@ def listen(self, agent_process: AgentProcess):
 
         return agent_process.get_response()
 
+    def parse_result(self, prompt):
+        """ each agent has their own parser """
+        pass
+
+    #######################
+    # getters and setters #
+    #######################
+
     def set_aid(self, aid):
         self.aid = aid
 

pyopenagi/agents/example/README.md

@@ -0,0 +1,3 @@
+# pyopenagi/agents/example
+
+Here are the example agents we created to demo agent creation in OpenAGI.

pyopenagi/agents/example/rag_agent/README.md

@@ -0,0 +1,14 @@
+# src/agents/agent_config
+
+Each agent holds a config file in addition to the class specifying what to run. The agent config is a JSON file.
+
+Each JSON file contains the following:
+
+1. `name` : name of the agent
+2. `description` : an array with one element containing the system prompt
+3. `workflow` : an array with plaintext describing what the agent will do at each iteration. this is fed into the LLM running the agent
+4. `tools` : an array with complex json objects
+- `type` : type of tool, typically "function"
+- `function` : if the type of function it contains data in the specific functions.
+
+For more detailed information, cite each specific agent as an example and fit it for your purposes.

pyopenagi/queues/README.md

@@ -0,0 +1,3 @@
+# pyopenagi/queues
+
+This contains implementations for queries to be passed to the LLM in a queue format so that we can have some waiting while one request is completing. 

pyopenagi/queues/base_queue.py

@@ -1,3 +1,5 @@
+# make a queue for agents to use
+
 import queue
 
 class BaseQueue:

pyopenagi/queues/llm_request_queue.py

@@ -1,3 +1,5 @@
+# same implementation as base_queue.py, but different class name
+
 from .base_queue import BaseQueue
 
 class LLMRequestQueue(BaseQueue):

pyopenagi/tools/README.md

@@ -0,0 +1,3 @@
+# pyopenagi/tools
+
+This is where all the tools are located. Each tool requires you to subclass the base tool and add the features required.

pyopenagi/utils/README.md

@@ -0,0 +1,7 @@
+# pyopenagi/utils
+
+Helper utils that are re-used in AIOS.
+
+These are various tools that we use in our internal implementations.
+
+In the future they shouldn't be copy pasted to AIOS.

pyopenagi/utils/compressor.py

@@ -1,3 +1,6 @@
+# uses zlibrary to compress data 
+# same as util in AIOS
+
 import zlib
 
 class Compressor:

pyopenagi/utils/logger.py

@@ -1,3 +1,6 @@
+# contains tools for logging actions
+# the thread scheduler, agents, and the LLM kernel of AIOS are all loggable here
+
 import click
 
 import os
@@ -34,6 +37,8 @@ def log_to_file(self, content, log_file):
             w.writelines(content)
 
 class SchedulerLogger(BaseLogger):
+    """ log the threads """
+
     def __init__(self, logger_name, log_mode="console") -> None:
         super().__init__(logger_name, log_mode)
         self.level_color = {
@@ -52,6 +57,8 @@ def load_log_file(self):
 
 
 class AgentLogger(BaseLogger):
+    """ log the agents """
+
     def __init__(self, logger_name, log_mode="console") -> None:
         super().__init__(logger_name, log_mode)
         self.level_color = {
@@ -71,6 +78,8 @@ def load_log_file(self):
 
 
 class LLMKernelLogger(BaseLogger):
+    """ log the LLM kernel """
+
     def __init__(self, logger_name, log_mode="console") -> None:
         super().__init__(logger_name, log_mode)
         self.level_color = {

pyopenagi/utils/utils.py

@@ -1,3 +1,6 @@
+# helper utils AIOS uses
+# file is not particularly organized but mainly small functions
+
 import argparse
 
 import os

tests/README.md

@@ -1,5 +1,7 @@
 # tests
 
-This directory contains quick tests of methods that we rely on heavily throughout this project. You can simply run the tests in the same manner.
+This directory contains tests you can use to test specific features of the project so you can figure out which specific parts work easier. 
 
-In the future, we intend to use error code to differentiate between successful and failed tests.
+For example, test_agent_creation.py simply tests the AgentProcess ability to hold agents.
+
+We want to use error code to differentiate between successful and failed tests in the future.

tests/test_agent_creation.py

@@ -1,3 +1,5 @@
+# make sure we can create agents
+
 from pyopenagi.agents.agent_process import AgentProcess
 from pyopenagi.utils.chat_template import Query