feat(integrations-service): Add new integrations & refactor integrati…

…ons service (#540)

Add new integrations for Brave, BrowserBase, and Spider, and refactor the integration service to support these changes.

  - **Integrations**:
    - Add new providers: `brave`, `browserbase`, and `spider` in `providers.py`.
    - Refactor existing providers: `wikipedia`, `weather`, `hacker_news`.
  - **Models**:
    - Add `BraveSearchSetup`, `BraveSearchArguments`, `BraveSearchOutput` in `brave.py`.
    - Add `BrowserBaseSetup`, `BrowserBaseLoadArguments`, `BrowserBaseLoadOutput` in `browserbase.py`.
    - Add `SpiderSetup`, `SpiderFetchArguments`, `SpiderFetchOutput` in `spider.py`.
    - Update `ExecutionRequest`, `ExecutionResponse` in `execution.py` to include new providers.
  - **Routers**:
    - Add `get_integration` in `get_integration.py`.
    - Add `get_integration_tool` in `get_integration_tool.py`.
    - Update `execute` in `execute.py` to handle new providers and methods.
  - **Utilities**:
    - Refactor `execute_integration` in `execute_integration.py` to dynamically load and execute provider methods.
    - Add integration logic for new providers in `integrations/utils/integrations/`.
  - **Misc**:
    - Update `pyproject.toml` to include new dependencies: `spider-client`, `browserbase`.
    - Update `scalars.tsp` and `models.tsp` to include new integration providers.

agents-api/agents_api/autogen/Tools.py

@@ -89,14 +89,18 @@ class IntegrationDef(BaseModel):
     model_config = ConfigDict(
         populate_by_name=True,
     )
-    provider: Literal[
-        "dummy",
-        "dalle_image_generator",
-        "duckduckgo_search",
-        "hacker_news",
-        "weather",
-        "wikipedia",
-    ]
+    provider: (
+        Literal[
+            "dummy",
+            "hacker_news",
+            "weather",
+            "wikipedia",
+            "spider",
+            "brave",
+            "browserbase",
+        ]
+        | str
+    )
     """
     The provider of the integration
     """
@@ -129,12 +133,14 @@ class IntegrationDefUpdate(BaseModel):
     provider: (
         Literal[
             "dummy",
-            "dalle_image_generator",
-            "duckduckgo_search",
             "hacker_news",
             "weather",
             "wikipedia",
+            "spider",
+            "brave",
+            "browserbase",
         ]
+        | str
         | None
     ) = None
     """

agents-api/agents_api/clients/integrations.py

@@ -19,7 +19,7 @@ async def run_integration_service(
     slug = f"{provider}/{method}" if method else provider
     url = f"{integration_service_url}/execute/{slug}"
 
-    setup = setup or {}
+    setup = setup or None
 
     async with AsyncClient() as client:
         response = await client.post(

integrations-service/integrations/models/__init__.py

@@ -1,17 +1,18 @@
-from .dalle_image_generator import (
-    DalleImageGeneratorArguments,
-    DalleImageGeneratorSetup,
+from .base_models import (
+    BaseArguments,
+    BaseOutput,
+    BaseProvider,
+    BaseProviderMethod,
+    BaseSetup,
+    ProviderInfo,
 )
-from .duckduckgo_search import DuckDuckGoSearchExecutionArguments
-from .hacker_news import HackerNewsExecutionArguments
-
-# TODO: Move these models somewhere else
-from .models import (
-    ExecuteIntegrationArguments,
-    ExecuteIntegrationSetup,
-    IntegrationDef,
-    IntegrationExecutionRequest,
-    IntegrationExecutionResponse,
+from .brave import BraveSearchArguments, BraveSearchOutput, BraveSearchSetup
+from .browserbase import (
+    BrowserBaseLoadArguments,
+    BrowserBaseLoadOutput,
+    BrowserBaseSetup,
 )
-from .weather import WeatherExecutionArguments, WeatherExecutionSetup
-from .wikipedia import WikipediaExecutionArguments
+from .hacker_news import HackerNewsFetchArguments, HackerNewsFetchOutput
+from .spider import SpiderFetchArguments, SpiderFetchOutput, SpiderSetup
+from .weather import WeatherGetArguments, WeatherGetOutput, WeatherSetup
+from .wikipedia import WikipediaSearchArguments, WikipediaSearchOutput

integrations-service/integrations/models/base_models.py

@@ -0,0 +1,36 @@
+from typing import Annotated, Any, Optional
+
+from pydantic import BaseModel, Field, RootModel
+from pydantic_core import Url
+
+IdentifierName = Annotated[str, Field(max_length=40, pattern="^[^\\W0-9]\\w*$")]
+
+
+class BaseSetup(BaseModel): ...
+
+
+class BaseArguments(BaseModel): ...
+
+
+class BaseOutput(BaseModel): ...
+
+
+class ProviderInfo(BaseModel):
+    url: Optional[Url]
+    docs: Optional[Url]
+    icon: Optional[Url]
+    friendly_name: str
+
+
+class BaseProviderMethod(BaseModel):
+    method: IdentifierName
+    description: str
+    arguments: type[BaseArguments]
+    output: type[BaseOutput]
+
+
+class BaseProvider(BaseModel):
+    provider: IdentifierName
+    setup: type[BaseSetup] | None
+    methods: list[BaseProviderMethod]
+    info: ProviderInfo

integrations-service/integrations/models/brave.py

@@ -0,0 +1,19 @@
+from pydantic import Field
+
+from .base_models import (
+    BaseArguments,
+    BaseOutput,
+    BaseSetup,
+)
+
+
+class BraveSearchSetup(BaseSetup):
+    api_key: str = Field(..., description="The api key for Brave Search")
+
+
+class BraveSearchArguments(BaseArguments):
+    query: str = Field(..., description="The search query for searching with Brave")
+
+
+class BraveSearchOutput(BaseOutput):
+    result: str = Field(..., description="The result of the Brave Search")

integrations-service/integrations/models/browserbase.py

@@ -0,0 +1,29 @@
+from typing import List, Optional
+
+from langchain_core.documents import Document
+from pydantic import Field
+from pydantic_core import Url
+
+from .base_models import (
+    BaseArguments,
+    BaseOutput,
+    BaseSetup,
+)
+
+
+class BrowserBaseSetup(BaseSetup):
+    api_key: str = Field(..., description="The api key for BrowserBase")
+    project_id: str = Field(..., description="The project id for BrowserBase")
+    session_id: Optional[str] = Field(
+        None, description="The session id for BrowserBase"
+    )
+
+
+class BrowserBaseLoadArguments(BaseArguments):
+    urls: List[Url] = Field(..., description="The urls for loading with BrowserBase")
+
+
+class BrowserBaseLoadOutput(BaseOutput):
+    documents: List[Document] = Field(
+        ..., description="The documents loaded from the urls"
+    )

integrations-service/integrations/models/execution.py

@@ -0,0 +1,50 @@
+from typing import Optional, Union
+
+from pydantic import BaseModel
+
+from .brave import BraveSearchArguments, BraveSearchOutput, BraveSearchSetup
+from .browserbase import (
+    BrowserBaseLoadArguments,
+    BrowserBaseLoadOutput,
+    BrowserBaseSetup,
+)
+from .hacker_news import HackerNewsFetchArguments, HackerNewsFetchOutput
+from .spider import SpiderFetchArguments, SpiderFetchOutput, SpiderSetup
+from .weather import WeatherGetArguments, WeatherGetOutput, WeatherSetup
+from .wikipedia import WikipediaSearchArguments, WikipediaSearchOutput
+
+ExecutionSetup = Union[
+    SpiderSetup,
+    WeatherSetup,
+    BraveSearchSetup,
+    BrowserBaseSetup,
+]
+
+ExecutionArguments = Union[
+    SpiderFetchArguments,
+    WeatherGetArguments,
+    HackerNewsFetchArguments,
+    WikipediaSearchArguments,
+    BraveSearchArguments,
+    BrowserBaseLoadArguments,
+]
+
+ExecutionResponse = Union[
+    SpiderFetchOutput,
+    WeatherGetOutput,
+    HackerNewsFetchOutput,
+    WikipediaSearchOutput,
+    BraveSearchOutput,
+    BrowserBaseLoadOutput,
+]
+
+
+class ExecutionRequest(BaseModel):
+    setup: Optional[ExecutionSetup]
+    """
+    The setup parameters the integration accepts (such as API keys)
+    """
+    arguments: ExecutionArguments
+    """
+    The arguments to pass to the integration
+    """

integrations-service/integrations/models/hacker_news.py

@@ -1,5 +1,15 @@
-from pydantic import BaseModel, Field
+from langchain_core.documents import Document
+from pydantic import Field
+from pydantic_core import Url
 
+from .base_models import BaseArguments, BaseOutput
 
-class HackerNewsExecutionArguments(BaseModel):
-    url: str = Field(..., description="The URL of the Hacker News thread to fetch")
+
+class HackerNewsFetchArguments(BaseArguments):
+    url: Url = Field(..., description="The URL of the Hacker News thread to fetch")
+
+
+class HackerNewsFetchOutput(BaseOutput):
+    documents: list[Document] = Field(
+        ..., description="The documents returned from the Hacker News search"
+    )

integrations-service/integrations/models/spider.py

@@ -0,0 +1,21 @@
+from langchain_core.documents import Document
+from pydantic import Field
+from pydantic_core import Url
+
+from .base_models import BaseArguments, BaseOutput, BaseSetup
+
+
+class SpiderSetup(BaseSetup):
+    spider_api_key: str = Field(..., description="The request for which to fetch data")
+
+
+class SpiderFetchArguments(BaseArguments):
+    url: Url = Field(..., description="The url for which to fetch data")
+    mode: str = Field("scrape", description="The type of crawlers")
+    params: dict | None = Field(None, description="The parameters for the Spider API")
+
+
+class SpiderFetchOutput(BaseOutput):
+    documents: list[Document] = Field(
+        ..., description="The documents returned from the spider"
+    )

integrations-service/integrations/models/weather.py

@@ -1,13 +1,23 @@
-from pydantic import BaseModel, Field
+from pydantic import Field
 
+from .base_models import (
+    BaseArguments,
+    BaseOutput,
+    BaseSetup,
+)
 
-class WeatherExecutionSetup(BaseModel):
+
+class WeatherSetup(BaseSetup):
     openweathermap_api_key: str = Field(
-        ..., description="The location for which to fetch weather data"
+        ..., description="The api key for OpenWeatherMap"
     )
 
 
-class WeatherExecutionArguments(BaseModel):
+class WeatherGetArguments(BaseArguments):
     location: str = Field(
         ..., description="The location for which to fetch weather data"
     )
+
+
+class WeatherGetOutput(BaseOutput):
+    result: str = Field(..., description="The weather data for the specified location")

integrations-service/integrations/models/wikipedia.py

@@ -1,6 +1,20 @@
-from pydantic import BaseModel, Field
+from typing import Literal
 
+from langchain_core.documents import Document
+from pydantic import Field
 
-class WikipediaExecutionArguments(BaseModel):
+from .base_models import (
+    BaseArguments,
+    BaseOutput,
+)
+
+
+class WikipediaSearchArguments(BaseArguments):
     query: str = Field(..., description="The search query string")
     load_max_docs: int = Field(2, description="Maximum number of documents to load")
+
+
+class WikipediaSearchOutput(BaseOutput):
+    documents: list[Document] = Field(
+        ..., description="The documents returned from the Wikipedia search"
+    )