fix(agents-api,integrations): Misc fixes and tenacity retry in integr…

…ations" (#710)

- **fix(integrations): General upkeep + tenacity retry**
- **feat(agents-api): Misc fixes in agents-api**

<!-- ELLIPSIS_HIDDEN -->


----

> [!IMPORTANT]
> Add `tenacity` retry logic to integration functions, remove unused
integrations, and improve agents-api and integrations services.
> 
>   - **Integrations**:
> - Add `tenacity` retry logic to `search()` in `brave.py`, `load()` in
`browserbase.py`, `send()` in `email.py`, `crawl()` in `spider.py`,
`get()` in `weather.py`, and `search()` in `wikipedia.py`.
> - Remove unused integrations: `dalle_image_generator.py`,
`duckduckgo_search.py`, `gmail/send_mail.py`, `hacker_news.py`, and
`request.py`.
>     - Remove `hacker_news` provider from `providers.py`.
>   - **Agents API**:
> - Update `log_step()` in `log_step.py` to include
`include_remote=True` in `context.model_dump()`.
>     - Add `NEWLINE` constant to `utils.py` and `template.py`.
> - Add `ApplicationError` to non-retryable exceptions in
`interceptors.py`.
> - Increase sleep time in `test_doc()` in `fixtures.py` from 0.1 to 0.5
seconds.
>   - **Misc**:
> - Remove unused imports in `base_models.py`, `email.py`,
`wikipedia.py`, `get_integration.py`, and `execute_integration.py`.
>     - Add `tenacity` to `pyproject.toml` dependencies.
> 
> <sup>This description was created by </sup>[<img alt="Ellipsis"
src="https://img.shields.io/badge/Ellipsis-blue?color=175173">](https://www.ellipsis.dev?ref=julep-ai%2Fjulep&utm_source=github&utm_medium=referral)<sup>
for 55fe24a. It will automatically
update as commits are pushed.</sup>


<!-- ELLIPSIS_HIDDEN -->

---------

Signed-off-by: Diwank Singh Tomer <[email protected]>

.github/workflows/translate-readme.yml

@@ -31,8 +31,15 @@ jobs:
           git add README-*.md
           git commit -m "chore(readme): translate README.md"
 
-      - name: Push changes
-        uses: ad-m/[email protected]
+      - name: Create Pull Request
+        uses: peter-evans/create-pull-request@v7
         with:
-          github_token: ${{ secrets.GITHUB_TOKEN }}
-          branch: ${{ github.ref }}
+          token: ${{ secrets.GITHUB_TOKEN }}
+          commit-message: "chore(readme): translate README.md"
+          title: "Translate README.md"
+          body: "This PR updates the translated versions of README.md"
+          branch: "translate-readme"
+          base: dev
+          add-paths: |
+            README.md
+            README-*.md

agents-api/agents_api/activities/task_steps/log_step.py

@@ -21,7 +21,9 @@ async def log_step(context: StepContext) -> StepOutcome:
 
         template: str = context.current_step.log
         output = await render_template(
-            template, context.model_dump(), skip_vars=["developer_id"]
+            template,
+            context.model_dump(include_remote=True),
+            skip_vars=["developer_id"],
         )
 
         result = StepOutcome(output=output)

agents-api/agents_api/activities/utils.py

@@ -201,13 +201,18 @@ class stdlib_statistics:
     "statistics": stdlib_statistics,
 }
 
+constants = {
+    "NEWLINE": "\n",
+}
+
 
 @beartype
 def get_evaluator(
     names: dict[str, Any], extra_functions: dict[str, Callable] | None = None
 ) -> SimpleEval:
     evaluator = EvalWithCompoundTypes(
-        names=names | stdlib, functions=ALLOWED_FUNCTIONS | (extra_functions or {})
+        names=names | stdlib | constants,
+        functions=ALLOWED_FUNCTIONS | (extra_functions or {}),
     )
 
     return evaluator

agents-api/agents_api/common/interceptors.py

@@ -46,6 +46,7 @@ async def execute_activity(self, input: ExecuteActivityInput):
             CompleteAsyncError,
             TemporalError,
             FailureError,
+            ApplicationError,
         ):
             raise
         except BaseException as e:
@@ -78,6 +79,7 @@ async def execute_workflow(self, input: ExecuteWorkflowInput):
             CompleteAsyncError,
             TemporalError,
             FailureError,
+            ApplicationError,
         ):
             raise
         except BaseException as e:

agents-api/agents_api/common/utils/template.py

@@ -37,7 +37,7 @@
 jinja_env.globals["true"] = True
 jinja_env.globals["false"] = False
 jinja_env.globals["null"] = None
-
+jinja_env.globals["NEWLINE"] = "\n"
 
 simple_jinja_regex = re.compile(r"{{|{%.+}}|%}", re.DOTALL)
 

agents-api/tests/fixtures.py

@@ -178,7 +178,7 @@ def test_doc(
         client=client,
     )
 
-    time.sleep(0.1)
+    time.sleep(0.5)
 
     yield doc
 

integrations-service/integrations/models/base_models.py

@@ -1,6 +1,6 @@
-from typing import Annotated, Any, Optional
+from typing import Annotated, Optional
 
-from pydantic import BaseModel, Field, RootModel
+from pydantic import BaseModel, Field
 from pydantic_core import Url
 
 IdentifierName = Annotated[str, Field(max_length=40, pattern="^[^\\W0-9]\\w*$")]

integrations-service/integrations/models/email.py

@@ -16,7 +16,9 @@ class EmailSetup(BaseSetup):
 
 class EmailArguments(BaseArguments):
     to: EmailStr = Field(..., description="The email address to send the email to")
-    from_: EmailStr = Field(..., alias="from", description="The email address to send the email from")
+    from_: EmailStr = Field(
+        ..., alias="from", description="The email address to send the email from"
+    )
     subject: str = Field(..., description="The subject of the email")
     body: str = Field(..., description="The body of the email")
 

integrations-service/integrations/models/wikipedia.py

@@ -1,5 +1,3 @@
-from typing import Literal
-
 from langchain_core.documents import Document
 from pydantic import Field
 

integrations-service/integrations/providers.py

@@ -10,8 +10,6 @@
     EmailArguments,
     EmailOutput,
     EmailSetup,
-    HackerNewsFetchArguments,
-    HackerNewsFetchOutput,
     ProviderInfo,
     SpiderFetchArguments,
     SpiderFetchOutput,
@@ -61,25 +59,6 @@
     ),
 )
 
-hacker_news = BaseProvider(
-    provider="hacker_news",
-    setup=None,
-    methods=[
-        BaseProviderMethod(
-            method="fetch",
-            description="Get the top stories from Hacker News",
-            arguments=HackerNewsFetchArguments,
-            output=HackerNewsFetchOutput,
-        ),
-    ],
-    info=ProviderInfo(
-        url="https://news.ycombinator.com/",
-        docs="https://news.ycombinator.com/newsguidelines.html",
-        icon="https://news.ycombinator.com/favicon.ico",
-        friendly_name="Hacker News",
-    ),
-)
-
 spider = BaseProvider(
     provider="spider",
     setup=SpiderSetup,
@@ -156,7 +135,6 @@
 providers = {
     "wikipedia": wikipedia,
     "weather": weather,
-    "hacker_news": hacker_news,
     "spider": spider,
     "brave": brave,
     "browserbase": browserbase,

integrations-service/integrations/routers/execution/execute.py

@@ -20,13 +20,13 @@ async def execute(
 
 
 @router.post("/execute/{provider}/{method}", tags=["execution"])
-def execute(
+async def execute(
     provider: IdentifierName,
     method: IdentifierName,
     data: ExecutionRequest,
 ) -> ExecutionResponse:
     try:
-        return execute_integration(
+        return await execute_integration(
             provider=provider, arguments=data.arguments, setup=data.setup, method=method
         )
     except ValueError as e:

integrations-service/integrations/routers/integrations/get_integration.py

@@ -1,5 +1,3 @@
-from typing import List
-
 from ...providers import providers
 from .router import router
 

integrations-service/integrations/utils/execute_integration.py

@@ -30,7 +30,7 @@ async def execute_integration(
             setup = setup_class(**setup.model_dump())
 
     arguments_class = next(m for m in provider.methods if m.method == method).arguments
-    
+
     if not isinstance(arguments, arguments_class):
         parsed_arguments = arguments_class(**arguments.model_dump())
     else:

integrations-service/integrations/utils/integrations/__init__.py

@@ -1,7 +1,6 @@
 from .brave import search
 from .browserbase import load
 from .email import send
-from .hacker_news import fetch
 from .spider import crawl
 from .weather import get
 from .wikipedia import search

integrations-service/integrations/utils/integrations/brave.py

@@ -1,8 +1,14 @@
 from langchain_community.tools import BraveSearch
+from tenacity import retry, stop_after_attempt, wait_exponential
 
 from ...models import BraveSearchArguments, BraveSearchOutput, BraveSearchSetup
 
 
+@retry(
+    wait=wait_exponential(multiplier=1, min=4, max=10),
+    reraise=True,
+    stop=stop_after_attempt(4),
+)
 async def search(
     setup: BraveSearchSetup, arguments: BraveSearchArguments
 ) -> BraveSearchOutput:

integrations-service/integrations/utils/integrations/browserbase.py

@@ -1,8 +1,14 @@
 from langchain_community.document_loaders import BrowserbaseLoader
+from tenacity import retry, stop_after_attempt, wait_exponential
 
 from ...models import BrowserBaseLoadArguments, BrowserBaseLoadOutput, BrowserBaseSetup
 
 
+@retry(
+    wait=wait_exponential(multiplier=1, min=4, max=10),
+    reraise=True,
+    stop=stop_after_attempt(3),
+)
 async def load(
     setup: BrowserBaseSetup, arguments: BrowserBaseLoadArguments
 ) -> BrowserBaseLoadOutput:

integrations-service/integrations/utils/integrations/email.py

@@ -1,15 +1,17 @@
 from email.message import EmailMessage
 from smtplib import SMTP
 
-from beartype import beartype
+from tenacity import retry, stop_after_attempt, wait_exponential
 
 from ...models import EmailArguments, EmailOutput, EmailSetup
 
 
-# @beartype
-async def send(
-    setup: EmailSetup, arguments: EmailArguments
-) -> EmailOutput:
+@retry(
+    wait=wait_exponential(multiplier=1, min=4, max=10),
+    reraise=True,
+    stop=stop_after_attempt(4),
+)
+async def send(setup: EmailSetup, arguments: EmailArguments) -> EmailOutput:
     """
     Sends an email with the provided details.
     """

integrations-service/integrations/utils/integrations/spider.py

@@ -1,9 +1,17 @@
 from langchain_community.document_loaders import SpiderLoader
+from tenacity import retry, stop_after_attempt, wait_exponential
 
 from ...models import SpiderFetchArguments, SpiderFetchOutput, SpiderSetup
 
 
-async def crawl(setup: SpiderSetup, arguments: SpiderFetchArguments) -> SpiderFetchOutput:
+@retry(
+    wait=wait_exponential(multiplier=1, min=4, max=10),
+    reraise=True,
+    stop=stop_after_attempt(4),
+)
+async def crawl(
+    setup: SpiderSetup, arguments: SpiderFetchArguments
+) -> SpiderFetchOutput:
     """
     Fetches data from a specified URL.
     """

integrations-service/integrations/utils/integrations/weather.py

@@ -1,8 +1,14 @@
 from langchain_community.utilities import OpenWeatherMapAPIWrapper
+from tenacity import retry, stop_after_attempt, wait_exponential
 
 from ...models import WeatherGetArguments, WeatherGetOutput, WeatherSetup
 
 
+@retry(
+    wait=wait_exponential(multiplier=1, min=4, max=10),
+    reraise=True,
+    stop=stop_after_attempt(4),
+)
 async def get(setup: WeatherSetup, arguments: WeatherGetArguments) -> WeatherGetOutput:
     """
     Fetches weather data for a specified location using OpenWeatherMap API.
@@ -20,4 +26,3 @@ async def get(setup: WeatherSetup, arguments: WeatherGetArguments) -> WeatherGet
     weather = OpenWeatherMapAPIWrapper(openweathermap_api_key=openweathermap_api_key)
     result = weather.run(location)
     return WeatherGetOutput(result=result)
-

integrations-service/integrations/utils/integrations/wikipedia.py

@@ -1,8 +1,14 @@
 from langchain_community.document_loaders import WikipediaLoader
+from tenacity import retry, stop_after_attempt, wait_exponential
 
 from ...models import WikipediaSearchArguments, WikipediaSearchOutput
 
 
+@retry(
+    wait=wait_exponential(multiplier=1, min=4, max=10),
+    reraise=True,
+    stop=stop_after_attempt(4),
+)
 def search(arguments: WikipediaSearchArguments) -> WikipediaSearchOutput:
     """
     Searches Wikipedia for a given query and returns formatted results.