fix(integrations): General upkeep + tenacity retry

Signed-off-by: Diwank Singh Tomer <[email protected]>

integrations-service/integrations/models/base_models.py

@@ -1,6 +1,6 @@
-from typing import Annotated, Any, Optional
+from typing import Annotated, Optional
 
-from pydantic import BaseModel, Field, RootModel
+from pydantic import BaseModel, Field
 from pydantic_core import Url
 
 IdentifierName = Annotated[str, Field(max_length=40, pattern="^[^\\W0-9]\\w*$")]

integrations-service/integrations/models/email.py

@@ -16,7 +16,9 @@ class EmailSetup(BaseSetup):
 
 class EmailArguments(BaseArguments):
     to: EmailStr = Field(..., description="The email address to send the email to")
-    from_: EmailStr = Field(..., alias="from", description="The email address to send the email from")
+    from_: EmailStr = Field(
+        ..., alias="from", description="The email address to send the email from"
+    )
     subject: str = Field(..., description="The subject of the email")
     body: str = Field(..., description="The body of the email")
 

integrations-service/integrations/models/wikipedia.py

@@ -1,5 +1,3 @@
-from typing import Literal
-
 from langchain_core.documents import Document
 from pydantic import Field
 

integrations-service/integrations/providers.py

@@ -10,8 +10,6 @@
     EmailArguments,
     EmailOutput,
     EmailSetup,
-    HackerNewsFetchArguments,
-    HackerNewsFetchOutput,
     ProviderInfo,
     SpiderFetchArguments,
     SpiderFetchOutput,
@@ -61,25 +59,6 @@
     ),
 )
 
-hacker_news = BaseProvider(
-    provider="hacker_news",
-    setup=None,
-    methods=[
-        BaseProviderMethod(
-            method="fetch",
-            description="Get the top stories from Hacker News",
-            arguments=HackerNewsFetchArguments,
-            output=HackerNewsFetchOutput,
-        ),
-    ],
-    info=ProviderInfo(
-        url="https://news.ycombinator.com/",
-        docs="https://news.ycombinator.com/newsguidelines.html",
-        icon="https://news.ycombinator.com/favicon.ico",
-        friendly_name="Hacker News",
-    ),
-)
-
 spider = BaseProvider(
     provider="spider",
     setup=SpiderSetup,
@@ -156,7 +135,6 @@
 providers = {
     "wikipedia": wikipedia,
     "weather": weather,
-    "hacker_news": hacker_news,
     "spider": spider,
     "brave": brave,
     "browserbase": browserbase,

integrations-service/integrations/routers/execution/execute.py

@@ -20,13 +20,13 @@ async def execute(
 
 
 @router.post("/execute/{provider}/{method}", tags=["execution"])
-def execute(
+async def execute(
     provider: IdentifierName,
     method: IdentifierName,
     data: ExecutionRequest,
 ) -> ExecutionResponse:
     try:
-        return execute_integration(
+        return await execute_integration(
             provider=provider, arguments=data.arguments, setup=data.setup, method=method
         )
     except ValueError as e:

integrations-service/integrations/routers/integrations/get_integration.py

@@ -1,5 +1,3 @@
-from typing import List
-
 from ...providers import providers
 from .router import router
 

integrations-service/integrations/utils/execute_integration.py

@@ -30,7 +30,7 @@ async def execute_integration(
             setup = setup_class(**setup.model_dump())
 
     arguments_class = next(m for m in provider.methods if m.method == method).arguments
-    
+
     if not isinstance(arguments, arguments_class):
         parsed_arguments = arguments_class(**arguments.model_dump())
     else:

integrations-service/integrations/utils/integrations/__init__.py

@@ -1,7 +1,6 @@
 from .brave import search
 from .browserbase import load
 from .email import send
-from .hacker_news import fetch
 from .spider import crawl
 from .weather import get
 from .wikipedia import search

integrations-service/integrations/utils/integrations/brave.py

@@ -1,8 +1,14 @@
 from langchain_community.tools import BraveSearch
+from tenacity import retry, stop_after_attempt, wait_exponential
 
 from ...models import BraveSearchArguments, BraveSearchOutput, BraveSearchSetup
 
 
+@retry(
+    wait=wait_exponential(multiplier=1, min=4, max=10),
+    reraise=True,
+    stop=stop_after_attempt(4),
+)
 async def search(
     setup: BraveSearchSetup, arguments: BraveSearchArguments
 ) -> BraveSearchOutput:

integrations-service/integrations/utils/integrations/browserbase.py

@@ -1,8 +1,14 @@
 from langchain_community.document_loaders import BrowserbaseLoader
+from tenacity import retry, stop_after_attempt, wait_exponential
 
 from ...models import BrowserBaseLoadArguments, BrowserBaseLoadOutput, BrowserBaseSetup
 
 
+@retry(
+    wait=wait_exponential(multiplier=1, min=4, max=10),
+    reraise=True,
+    stop=stop_after_attempt(3),
+)
 async def load(
     setup: BrowserBaseSetup, arguments: BrowserBaseLoadArguments
 ) -> BrowserBaseLoadOutput:

integrations-service/integrations/utils/integrations/email.py

@@ -1,15 +1,17 @@
 from email.message import EmailMessage
 from smtplib import SMTP
 
-from beartype import beartype
+from tenacity import retry, stop_after_attempt, wait_exponential
 
 from ...models import EmailArguments, EmailOutput, EmailSetup
 
 
-# @beartype
-async def send(
-    setup: EmailSetup, arguments: EmailArguments
-) -> EmailOutput:
+@retry(
+    wait=wait_exponential(multiplier=1, min=4, max=10),
+    reraise=True,
+    stop=stop_after_attempt(4),
+)
+async def send(setup: EmailSetup, arguments: EmailArguments) -> EmailOutput:
     """
     Sends an email with the provided details.
     """

integrations-service/integrations/utils/integrations/spider.py

@@ -1,9 +1,17 @@
 from langchain_community.document_loaders import SpiderLoader
+from tenacity import retry, stop_after_attempt, wait_exponential
 
 from ...models import SpiderFetchArguments, SpiderFetchOutput, SpiderSetup
 
 
-async def crawl(setup: SpiderSetup, arguments: SpiderFetchArguments) -> SpiderFetchOutput:
+@retry(
+    wait=wait_exponential(multiplier=1, min=4, max=10),
+    reraise=True,
+    stop=stop_after_attempt(4),
+)
+async def crawl(
+    setup: SpiderSetup, arguments: SpiderFetchArguments
+) -> SpiderFetchOutput:
     """
     Fetches data from a specified URL.
     """

integrations-service/integrations/utils/integrations/weather.py

@@ -1,8 +1,14 @@
 from langchain_community.utilities import OpenWeatherMapAPIWrapper
+from tenacity import retry, stop_after_attempt, wait_exponential
 
 from ...models import WeatherGetArguments, WeatherGetOutput, WeatherSetup
 
 
+@retry(
+    wait=wait_exponential(multiplier=1, min=4, max=10),
+    reraise=True,
+    stop=stop_after_attempt(4),
+)
 async def get(setup: WeatherSetup, arguments: WeatherGetArguments) -> WeatherGetOutput:
     """
     Fetches weather data for a specified location using OpenWeatherMap API.
@@ -20,4 +26,3 @@ async def get(setup: WeatherSetup, arguments: WeatherGetArguments) -> WeatherGet
     weather = OpenWeatherMapAPIWrapper(openweathermap_api_key=openweathermap_api_key)
     result = weather.run(location)
     return WeatherGetOutput(result=result)
-

integrations-service/integrations/utils/integrations/wikipedia.py

@@ -1,8 +1,14 @@
 from langchain_community.document_loaders import WikipediaLoader
+from tenacity import retry, stop_after_attempt, wait_exponential
 
 from ...models import WikipediaSearchArguments, WikipediaSearchOutput
 
 
+@retry(
+    wait=wait_exponential(multiplier=1, min=4, max=10),
+    reraise=True,
+    stop=stop_after_attempt(4),
+)
 def search(arguments: WikipediaSearchArguments) -> WikipediaSearchOutput:
     """
     Searches Wikipedia for a given query and returns formatted results.

integrations-service/pyproject.toml

@@ -21,6 +21,7 @@ spider-client = "^0.0.70"
 browserbase = "^0.3.0"
 setuptools = "^75.1.0"
 beartype = "^0.19.0"
+tenacity = "^9.0.0"
 
 [tool.poe.tasks]
 format = "ruff format"