From ffd0f76699d212c760518c1745a7ca8fed12e257 Mon Sep 17 00:00:00 2001 From: Diwank Singh Tomer Date: Sat, 19 Oct 2024 00:17:32 -0400 Subject: [PATCH] fix(integrations): General upkeep + tenacity retry Signed-off-by: Diwank Singh Tomer --- .../integrations/models/base_models.py | 4 +- .../integrations/models/email.py | 4 +- .../integrations/models/wikipedia.py | 2 - .../integrations/providers.py | 22 ----------- .../integrations/routers/execution/execute.py | 4 +- .../routers/integrations/get_integration.py | 2 - .../integrations/utils/execute_integration.py | 2 +- .../utils/integrations/__init__.py | 1 - .../integrations/utils/integrations/brave.py | 6 +++ .../utils/integrations/browserbase.py | 6 +++ .../integrations/dalle_image_generator.py | 22 ----------- .../utils/integrations/duckduckgo_search.py | 15 -------- .../integrations/utils/integrations/email.py | 12 +++--- .../utils/integrations/gmail/send_mail.py | 6 --- .../utils/integrations/hacker_news.py | 22 ----------- .../utils/integrations/request.py | 0 .../integrations/utils/integrations/spider.py | 10 ++++- .../utils/integrations/weather.py | 7 +++- .../utils/integrations/wikipedia.py | 6 +++ integrations-service/poetry.lock | 38 +++++++++---------- integrations-service/pyproject.toml | 1 + 21 files changed, 68 insertions(+), 124 deletions(-) delete mode 100644 integrations-service/integrations/utils/integrations/dalle_image_generator.py delete mode 100644 integrations-service/integrations/utils/integrations/duckduckgo_search.py delete mode 100644 integrations-service/integrations/utils/integrations/gmail/send_mail.py delete mode 100644 integrations-service/integrations/utils/integrations/hacker_news.py delete mode 100644 integrations-service/integrations/utils/integrations/request.py diff --git a/integrations-service/integrations/models/base_models.py b/integrations-service/integrations/models/base_models.py index 9f119d460..55b614a2a 100644 --- a/integrations-service/integrations/models/base_models.py +++ b/integrations-service/integrations/models/base_models.py @@ -1,6 +1,6 @@ -from typing import Annotated, Any, Optional +from typing import Annotated, Optional -from pydantic import BaseModel, Field, RootModel +from pydantic import BaseModel, Field from pydantic_core import Url IdentifierName = Annotated[str, Field(max_length=40, pattern="^[^\\W0-9]\\w*$")] diff --git a/integrations-service/integrations/models/email.py b/integrations-service/integrations/models/email.py index cab794a7d..11943968d 100644 --- a/integrations-service/integrations/models/email.py +++ b/integrations-service/integrations/models/email.py @@ -16,7 +16,9 @@ class EmailSetup(BaseSetup): class EmailArguments(BaseArguments): to: EmailStr = Field(..., description="The email address to send the email to") - from_: EmailStr = Field(..., alias="from", description="The email address to send the email from") + from_: EmailStr = Field( + ..., alias="from", description="The email address to send the email from" + ) subject: str = Field(..., description="The subject of the email") body: str = Field(..., description="The body of the email") diff --git a/integrations-service/integrations/models/wikipedia.py b/integrations-service/integrations/models/wikipedia.py index 8c8e4f623..36a2108ce 100644 --- a/integrations-service/integrations/models/wikipedia.py +++ b/integrations-service/integrations/models/wikipedia.py @@ -1,5 +1,3 @@ -from typing import Literal - from langchain_core.documents import Document from pydantic import Field diff --git a/integrations-service/integrations/providers.py b/integrations-service/integrations/providers.py index 41b5bf757..76ede47af 100644 --- a/integrations-service/integrations/providers.py +++ b/integrations-service/integrations/providers.py @@ -10,8 +10,6 @@ EmailArguments, EmailOutput, EmailSetup, - HackerNewsFetchArguments, - HackerNewsFetchOutput, ProviderInfo, SpiderFetchArguments, SpiderFetchOutput, @@ -61,25 +59,6 @@ ), ) -hacker_news = BaseProvider( - provider="hacker_news", - setup=None, - methods=[ - BaseProviderMethod( - method="fetch", - description="Get the top stories from Hacker News", - arguments=HackerNewsFetchArguments, - output=HackerNewsFetchOutput, - ), - ], - info=ProviderInfo( - url="https://news.ycombinator.com/", - docs="https://news.ycombinator.com/newsguidelines.html", - icon="https://news.ycombinator.com/favicon.ico", - friendly_name="Hacker News", - ), -) - spider = BaseProvider( provider="spider", setup=SpiderSetup, @@ -156,7 +135,6 @@ providers = { "wikipedia": wikipedia, "weather": weather, - "hacker_news": hacker_news, "spider": spider, "brave": brave, "browserbase": browserbase, diff --git a/integrations-service/integrations/routers/execution/execute.py b/integrations-service/integrations/routers/execution/execute.py index df4bf913a..3097c6c99 100644 --- a/integrations-service/integrations/routers/execution/execute.py +++ b/integrations-service/integrations/routers/execution/execute.py @@ -20,13 +20,13 @@ async def execute( @router.post("/execute/{provider}/{method}", tags=["execution"]) -def execute( +async def execute( provider: IdentifierName, method: IdentifierName, data: ExecutionRequest, ) -> ExecutionResponse: try: - return execute_integration( + return await execute_integration( provider=provider, arguments=data.arguments, setup=data.setup, method=method ) except ValueError as e: diff --git a/integrations-service/integrations/routers/integrations/get_integration.py b/integrations-service/integrations/routers/integrations/get_integration.py index 2a9b34595..7b45c8190 100644 --- a/integrations-service/integrations/routers/integrations/get_integration.py +++ b/integrations-service/integrations/routers/integrations/get_integration.py @@ -1,5 +1,3 @@ -from typing import List - from ...providers import providers from .router import router diff --git a/integrations-service/integrations/utils/execute_integration.py b/integrations-service/integrations/utils/execute_integration.py index ab1907885..f1dd1965a 100644 --- a/integrations-service/integrations/utils/execute_integration.py +++ b/integrations-service/integrations/utils/execute_integration.py @@ -30,7 +30,7 @@ async def execute_integration( setup = setup_class(**setup.model_dump()) arguments_class = next(m for m in provider.methods if m.method == method).arguments - + if not isinstance(arguments, arguments_class): parsed_arguments = arguments_class(**arguments.model_dump()) else: diff --git a/integrations-service/integrations/utils/integrations/__init__.py b/integrations-service/integrations/utils/integrations/__init__.py index 238dacfd3..ce5ed0e83 100644 --- a/integrations-service/integrations/utils/integrations/__init__.py +++ b/integrations-service/integrations/utils/integrations/__init__.py @@ -1,7 +1,6 @@ from .brave import search from .browserbase import load from .email import send -from .hacker_news import fetch from .spider import crawl from .weather import get from .wikipedia import search diff --git a/integrations-service/integrations/utils/integrations/brave.py b/integrations-service/integrations/utils/integrations/brave.py index 10bfe8084..3723f80d7 100644 --- a/integrations-service/integrations/utils/integrations/brave.py +++ b/integrations-service/integrations/utils/integrations/brave.py @@ -1,8 +1,14 @@ from langchain_community.tools import BraveSearch +from tenacity import retry, stop_after_attempt, wait_exponential from ...models import BraveSearchArguments, BraveSearchOutput, BraveSearchSetup +@retry( + wait=wait_exponential(multiplier=1, min=4, max=10), + reraise=True, + stop=stop_after_attempt(4), +) async def search( setup: BraveSearchSetup, arguments: BraveSearchArguments ) -> BraveSearchOutput: diff --git a/integrations-service/integrations/utils/integrations/browserbase.py b/integrations-service/integrations/utils/integrations/browserbase.py index 7cc672662..87e7c8cac 100644 --- a/integrations-service/integrations/utils/integrations/browserbase.py +++ b/integrations-service/integrations/utils/integrations/browserbase.py @@ -1,8 +1,14 @@ from langchain_community.document_loaders import BrowserbaseLoader +from tenacity import retry, stop_after_attempt, wait_exponential from ...models import BrowserBaseLoadArguments, BrowserBaseLoadOutput, BrowserBaseSetup +@retry( + wait=wait_exponential(multiplier=1, min=4, max=10), + reraise=True, + stop=stop_after_attempt(3), +) async def load( setup: BrowserBaseSetup, arguments: BrowserBaseLoadArguments ) -> BrowserBaseLoadOutput: diff --git a/integrations-service/integrations/utils/integrations/dalle_image_generator.py b/integrations-service/integrations/utils/integrations/dalle_image_generator.py deleted file mode 100644 index e0a6496b8..000000000 --- a/integrations-service/integrations/utils/integrations/dalle_image_generator.py +++ /dev/null @@ -1,22 +0,0 @@ -from langchain_community.utilities.dalle_image_generator import DallEAPIWrapper - -from ...models import DalleImageGeneratorArguments, DalleImageGeneratorSetup - - -async def dalle_image_generator( - setup: DalleImageGeneratorSetup, arguments: DalleImageGeneratorArguments -) -> str: - """ - Generates an image using DALL-E based on a provided prompt. - """ - - assert isinstance(setup, DalleImageGeneratorSetup), "Invalid setup" - assert isinstance(arguments, DalleImageGeneratorArguments), "Invalid arguments" - - # FIXME: Fix OpenAI API Key error - - dalle = DallEAPIWrapper(api_key=setup.api_key) - prompt = arguments.prompt - if not prompt: - raise ValueError("Prompt parameter is required for DALL-E image generation") - return dalle.run(prompt) diff --git a/integrations-service/integrations/utils/integrations/duckduckgo_search.py b/integrations-service/integrations/utils/integrations/duckduckgo_search.py deleted file mode 100644 index a8ea38f56..000000000 --- a/integrations-service/integrations/utils/integrations/duckduckgo_search.py +++ /dev/null @@ -1,15 +0,0 @@ -from langchain_community.tools import DuckDuckGoSearchRun - -from ...models import DuckDuckGoSearchExecutionArguments - - -async def duckduckgo_search(arguments: DuckDuckGoSearchExecutionArguments) -> str: - """ - Performs a web search using DuckDuckGo and returns the results. - """ - - search = DuckDuckGoSearchRun() - query = arguments.query - if not query: - raise ValueError("Query parameter is required for DuckDuckGo search") - return search.run(query) diff --git a/integrations-service/integrations/utils/integrations/email.py b/integrations-service/integrations/utils/integrations/email.py index b0ced9ff5..5913beb73 100644 --- a/integrations-service/integrations/utils/integrations/email.py +++ b/integrations-service/integrations/utils/integrations/email.py @@ -1,15 +1,17 @@ from email.message import EmailMessage from smtplib import SMTP -from beartype import beartype +from tenacity import retry, stop_after_attempt, wait_exponential from ...models import EmailArguments, EmailOutput, EmailSetup -# @beartype -async def send( - setup: EmailSetup, arguments: EmailArguments -) -> EmailOutput: +@retry( + wait=wait_exponential(multiplier=1, min=4, max=10), + reraise=True, + stop=stop_after_attempt(4), +) +async def send(setup: EmailSetup, arguments: EmailArguments) -> EmailOutput: """ Sends an email with the provided details. """ diff --git a/integrations-service/integrations/utils/integrations/gmail/send_mail.py b/integrations-service/integrations/utils/integrations/gmail/send_mail.py deleted file mode 100644 index e1863f280..000000000 --- a/integrations-service/integrations/utils/integrations/gmail/send_mail.py +++ /dev/null @@ -1,6 +0,0 @@ -async def send_mail(arguments: dict) -> str: - """ - Dummy integration for sending an email to a specified recipient with a given subject and message. - """ - - return "Mail sent" diff --git a/integrations-service/integrations/utils/integrations/hacker_news.py b/integrations-service/integrations/utils/integrations/hacker_news.py deleted file mode 100644 index 526024c72..000000000 --- a/integrations-service/integrations/utils/integrations/hacker_news.py +++ /dev/null @@ -1,22 +0,0 @@ -from langchain_community.document_loaders import HNLoader - -from ...models import HackerNewsFetchArguments, HackerNewsFetchOutput - - -async def fetch(arguments: HackerNewsFetchArguments) -> HackerNewsFetchOutput: - """ - Fetches and formats content from a Hacker News thread using the provided URL. - """ - - assert isinstance(arguments, HackerNewsFetchArguments), "Invalid arguments" - - url = arguments.url - if not url: - raise ValueError("URL parameter is required for Hacker News search") - loader = HNLoader(str(url)) - documents = loader.load() - - if not documents: - raise ValueError("No data found for the given URL") - - return HackerNewsFetchOutput(documents=documents) diff --git a/integrations-service/integrations/utils/integrations/request.py b/integrations-service/integrations/utils/integrations/request.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/integrations-service/integrations/utils/integrations/spider.py b/integrations-service/integrations/utils/integrations/spider.py index a355e2347..7ff06a46d 100644 --- a/integrations-service/integrations/utils/integrations/spider.py +++ b/integrations-service/integrations/utils/integrations/spider.py @@ -1,9 +1,17 @@ from langchain_community.document_loaders import SpiderLoader +from tenacity import retry, stop_after_attempt, wait_exponential from ...models import SpiderFetchArguments, SpiderFetchOutput, SpiderSetup -async def crawl(setup: SpiderSetup, arguments: SpiderFetchArguments) -> SpiderFetchOutput: +@retry( + wait=wait_exponential(multiplier=1, min=4, max=10), + reraise=True, + stop=stop_after_attempt(4), +) +async def crawl( + setup: SpiderSetup, arguments: SpiderFetchArguments +) -> SpiderFetchOutput: """ Fetches data from a specified URL. """ diff --git a/integrations-service/integrations/utils/integrations/weather.py b/integrations-service/integrations/utils/integrations/weather.py index e9393bc09..0266aca4b 100644 --- a/integrations-service/integrations/utils/integrations/weather.py +++ b/integrations-service/integrations/utils/integrations/weather.py @@ -1,8 +1,14 @@ from langchain_community.utilities import OpenWeatherMapAPIWrapper +from tenacity import retry, stop_after_attempt, wait_exponential from ...models import WeatherGetArguments, WeatherGetOutput, WeatherSetup +@retry( + wait=wait_exponential(multiplier=1, min=4, max=10), + reraise=True, + stop=stop_after_attempt(4), +) async def get(setup: WeatherSetup, arguments: WeatherGetArguments) -> WeatherGetOutput: """ Fetches weather data for a specified location using OpenWeatherMap API. @@ -20,4 +26,3 @@ async def get(setup: WeatherSetup, arguments: WeatherGetArguments) -> WeatherGet weather = OpenWeatherMapAPIWrapper(openweathermap_api_key=openweathermap_api_key) result = weather.run(location) return WeatherGetOutput(result=result) - diff --git a/integrations-service/integrations/utils/integrations/wikipedia.py b/integrations-service/integrations/utils/integrations/wikipedia.py index aa53b5515..d4b212d7b 100644 --- a/integrations-service/integrations/utils/integrations/wikipedia.py +++ b/integrations-service/integrations/utils/integrations/wikipedia.py @@ -1,8 +1,14 @@ from langchain_community.document_loaders import WikipediaLoader +from tenacity import retry, stop_after_attempt, wait_exponential from ...models import WikipediaSearchArguments, WikipediaSearchOutput +@retry( + wait=wait_exponential(multiplier=1, min=4, max=10), + reraise=True, + stop=stop_after_attempt(4), +) def search(arguments: WikipediaSearchArguments) -> WikipediaSearchOutput: """ Searches Wikipedia for a given query and returns formatted results. diff --git a/integrations-service/poetry.lock b/integrations-service/poetry.lock index bc294a56f..83a500a29 100644 --- a/integrations-service/poetry.lock +++ b/integrations-service/poetry.lock @@ -889,18 +889,18 @@ files = [ [[package]] name = "langchain" -version = "0.3.2" +version = "0.3.4" description = "Building applications with LLMs through composability" optional = false python-versions = "<4.0,>=3.9" files = [ - {file = "langchain-0.3.2-py3-none-any.whl", hash = "sha256:cf005dcba132e46fb5e8d3dfaf7f8751bffd2d73e738c36be58f41edc7e3a4b8"}, - {file = "langchain-0.3.2.tar.gz", hash = "sha256:dc330e6eb10d81d23ba0305d18358702c73cc59e95c410eca6c6779aab4ddc9b"}, + {file = "langchain-0.3.4-py3-none-any.whl", hash = "sha256:7a1241d9429510d2083c62df0da998a7b2b05c730cd4255b89da9d47c57f48fd"}, + {file = "langchain-0.3.4.tar.gz", hash = "sha256:3596515fcd0157dece6ec96e0240d29f4cf542d91ecffc815d32e35198dfff37"}, ] [package.dependencies] aiohttp = ">=3.8.3,<4.0.0" -langchain-core = ">=0.3.8,<0.4.0" +langchain-core = ">=0.3.12,<0.4.0" langchain-text-splitters = ">=0.3.0,<0.4.0" langsmith = ">=0.1.17,<0.2.0" numpy = {version = ">=1.26.0,<2.0.0", markers = "python_version >= \"3.12\""} @@ -908,41 +908,41 @@ pydantic = ">=2.7.4,<3.0.0" PyYAML = ">=5.3" requests = ">=2,<3" SQLAlchemy = ">=1.4,<3" -tenacity = ">=8.1.0,<8.4.0 || >8.4.0,<9.0.0" +tenacity = ">=8.1.0,<8.4.0 || >8.4.0,<10" [[package]] name = "langchain-community" -version = "0.3.1" +version = "0.3.3" description = "Community contributed LangChain integrations." optional = false python-versions = "<4.0,>=3.9" files = [ - {file = "langchain_community-0.3.1-py3-none-any.whl", hash = "sha256:627eb26c16417764762ac47dd0d3005109f750f40242a88bb8f2958b798bcf90"}, - {file = "langchain_community-0.3.1.tar.gz", hash = "sha256:c964a70628f266a61647e58f2f0434db633d4287a729f100a81dd8b0654aec93"}, + {file = "langchain_community-0.3.3-py3-none-any.whl", hash = "sha256:319cfc2f923a066c91fbb8e02decd7814018af952b6b98298b8ac9d30ea1da56"}, + {file = "langchain_community-0.3.3.tar.gz", hash = "sha256:bfb3f2b219aed21087e0ecb7d2ebd1c81401c02b92239e11645c822d5be63f80"}, ] [package.dependencies] aiohttp = ">=3.8.3,<4.0.0" dataclasses-json = ">=0.5.7,<0.7" -langchain = ">=0.3.1,<0.4.0" -langchain-core = ">=0.3.6,<0.4.0" +langchain = ">=0.3.4,<0.4.0" +langchain-core = ">=0.3.12,<0.4.0" langsmith = ">=0.1.125,<0.2.0" numpy = {version = ">=1.26.0,<2.0.0", markers = "python_version >= \"3.12\""} pydantic-settings = ">=2.4.0,<3.0.0" PyYAML = ">=5.3" requests = ">=2,<3" SQLAlchemy = ">=1.4,<3" -tenacity = ">=8.1.0,<8.4.0 || >8.4.0,<9.0.0" +tenacity = ">=8.1.0,<8.4.0 || >8.4.0,<10" [[package]] name = "langchain-core" -version = "0.3.9" +version = "0.3.12" description = "Building applications with LLMs through composability" optional = false python-versions = "<4.0,>=3.9" files = [ - {file = "langchain_core-0.3.9-py3-none-any.whl", hash = "sha256:26efa048666c7de56d0ab311de2c0778b04cbb2ffe95bff76139118f13815d01"}, - {file = "langchain_core-0.3.9.tar.gz", hash = "sha256:7a6ac988d24d0ddce5874b28f538cd95f69f502b7f50581de22aca0dc58199a8"}, + {file = "langchain_core-0.3.12-py3-none-any.whl", hash = "sha256:46050d34f5fa36dc57dca971c6a26f505643dd05ee0492c7ac286d0a78a82037"}, + {file = "langchain_core-0.3.12.tar.gz", hash = "sha256:98a3c078e375786aa84939bfd1111263af2f3bc402bbe2cac9fa18a387459cf2"}, ] [package.dependencies] @@ -954,7 +954,7 @@ pydantic = [ {version = ">=2.7.4,<3.0.0", markers = "python_full_version >= \"3.12.4\""}, ] PyYAML = ">=5.3" -tenacity = ">=8.1.0,<8.4.0 || >8.4.0,<9.0.0" +tenacity = ">=8.1.0,<8.4.0 || >8.4.0,<10.0.0" typing-extensions = ">=4.7" [[package]] @@ -2198,13 +2198,13 @@ widechars = ["wcwidth"] [[package]] name = "tenacity" -version = "8.5.0" +version = "9.0.0" description = "Retry code until it succeeds" optional = false python-versions = ">=3.8" files = [ - {file = "tenacity-8.5.0-py3-none-any.whl", hash = "sha256:b594c2a5945830c267ce6b79a166228323ed52718f30302c1359836112346687"}, - {file = "tenacity-8.5.0.tar.gz", hash = "sha256:8bc6c0c8a09b31e6cad13c47afbed1a567518250a9a171418582ed8d9c20ca78"}, + {file = "tenacity-9.0.0-py3-none-any.whl", hash = "sha256:93de0c98785b27fcf659856aa9f54bfbd399e29969b0621bc7f762bd441b4539"}, + {file = "tenacity-9.0.0.tar.gz", hash = "sha256:807f37ca97d62aa361264d497b0e31e92b8027044942bfa756160d908320d73b"}, ] [package.extras] @@ -2462,4 +2462,4 @@ multidict = ">=4.0" [metadata] lock-version = "2.0" python-versions = ">=3.12,<3.13" -content-hash = "4f933143b7f17beaa1c4e8bdc117978de07df01f9b8b0eb4eb11f88182230ee5" +content-hash = "6ea8ddd02f0921f3edb5527dce0abc6d174124516d4b7348ba4083e3737957fc" diff --git a/integrations-service/pyproject.toml b/integrations-service/pyproject.toml index 33173f2c2..592206fc6 100644 --- a/integrations-service/pyproject.toml +++ b/integrations-service/pyproject.toml @@ -21,6 +21,7 @@ spider-client = "^0.0.70" browserbase = "^0.3.0" setuptools = "^75.1.0" beartype = "^0.19.0" +tenacity = "^9.0.0" [tool.poe.tasks] format = "ruff format"