diff --git a/.gitignore b/.gitignore index f9ce2fae..015a1994 100644 --- a/.gitignore +++ b/.gitignore @@ -14,6 +14,8 @@ dist/ *.egg-info/ *.egg MANIFEST +*.python-version +*.lock docs/build/ docs/source/_templates/ @@ -31,6 +33,5 @@ examples/graph_examples/ScrapeGraphAI_generated_graph examples/**/result.csv examples/**/result.json main.py -*.python-version -*.lock + \ No newline at end of file diff --git a/.python-version b/.python-version deleted file mode 100644 index 8e34c813..00000000 --- a/.python-version +++ /dev/null @@ -1 +0,0 @@ -3.9.19 diff --git a/requirements-dev.lock b/requirements-dev.lock deleted file mode 100644 index 7c37321b..00000000 --- a/requirements-dev.lock +++ /dev/null @@ -1,311 +0,0 @@ -# generated by rye -# use `rye lock` or `rye sync` to update this lockfile -# -# last locked with the following flags: -# pre: false -# features: [] -# all-features: false -# with-sources: false - --e file:. -aiohttp==3.9.5 - # via langchain - # via langchain-community -aiosignal==1.3.1 - # via aiohttp -annotated-types==0.6.0 - # via pydantic -anthropic==0.25.9 - # via langchain-anthropic -anyio==4.3.0 - # via anthropic - # via groq - # via httpx - # via openai -async-timeout==4.0.3 - # via aiohttp - # via langchain -attrs==23.2.0 - # via aiohttp -beautifulsoup4==4.12.3 - # via google - # via scrapegraphai -boto3==1.34.105 - # via langchain-aws -botocore==1.34.105 - # via boto3 - # via s3transfer -cachetools==5.3.3 - # via google-auth -certifi==2024.2.2 - # via httpcore - # via httpx - # via requests -charset-normalizer==3.3.2 - # via requests -dataclasses-json==0.6.6 - # via langchain - # via langchain-community -defusedxml==0.7.1 - # via langchain-anthropic -distro==1.9.0 - # via anthropic - # via groq - # via openai -exceptiongroup==1.2.1 - # via anyio - # via pytest -faiss-cpu==1.8.0 - # via scrapegraphai -filelock==3.14.0 - # via huggingface-hub -free-proxy==1.1.1 - # via scrapegraphai -frozenlist==1.4.1 - # via aiohttp - # via aiosignal -fsspec==2024.3.1 - # via huggingface-hub -google==3.0.0 - # via scrapegraphai -google-ai-generativelanguage==0.6.3 - # via google-generativeai -google-api-core==2.19.0 - # via google-ai-generativelanguage - # via google-api-python-client - # via google-generativeai -google-api-python-client==2.129.0 - # via google-generativeai -google-auth==2.29.0 - # via google-ai-generativelanguage - # via google-api-core - # via google-api-python-client - # via google-auth-httplib2 - # via google-generativeai -google-auth-httplib2==0.2.0 - # via google-api-python-client -google-generativeai==0.5.3 - # via langchain-google-genai -googleapis-common-protos==1.63.0 - # via google-api-core - # via grpcio-status -graphviz==0.20.3 - # via scrapegraphai -greenlet==3.0.3 - # via playwright - # via sqlalchemy -groq==0.5.0 - # via langchain-groq -grpcio==1.63.0 - # via google-api-core - # via grpcio-status -grpcio-status==1.62.2 - # via google-api-core -h11==0.14.0 - # via httpcore -html2text==2024.2.26 - # via scrapegraphai -httpcore==1.0.5 - # via httpx -httplib2==0.22.0 - # via google-api-python-client - # via google-auth-httplib2 -httpx==0.27.0 - # via anthropic - # via groq - # via openai - # via yahoo-search-py -huggingface-hub==0.23.0 - # via tokenizers -idna==3.7 - # via anyio - # via httpx - # via requests - # via yarl -iniconfig==2.0.0 - # via pytest -jmespath==1.0.1 - # via boto3 - # via botocore -jsonpatch==1.33 - # via langchain - # via langchain-core -jsonpointer==2.4 - # via jsonpatch -langchain==0.1.15 - # via scrapegraphai -langchain-anthropic==0.1.11 - # via scrapegraphai -langchain-aws==0.1.3 - # via scrapegraphai -langchain-community==0.0.38 - # via langchain -langchain-core==0.1.52 - # via langchain - # via langchain-anthropic - # via langchain-aws - # via langchain-community - # via langchain-google-genai - # via langchain-groq - # via langchain-openai - # via langchain-text-splitters -langchain-google-genai==1.0.3 - # via scrapegraphai -langchain-groq==0.1.3 - # via scrapegraphai -langchain-openai==0.1.6 - # via scrapegraphai -langchain-text-splitters==0.0.1 - # via langchain -langsmith==0.1.57 - # via langchain - # via langchain-community - # via langchain-core -lxml==5.2.2 - # via free-proxy -marshmallow==3.21.2 - # via dataclasses-json -minify-html==0.15.0 - # via scrapegraphai -multidict==6.0.5 - # via aiohttp - # via yarl -mypy-extensions==1.0.0 - # via typing-inspect -numpy==1.26.4 - # via faiss-cpu - # via langchain - # via langchain-aws - # via langchain-community - # via pandas -openai==1.30.1 - # via langchain-openai -orjson==3.10.3 - # via langsmith -packaging==23.2 - # via huggingface-hub - # via langchain-core - # via marshmallow - # via pytest -pandas==2.2.2 - # via scrapegraphai -playwright==1.43.0 - # via scrapegraphai -pluggy==1.5.0 - # via pytest -proto-plus==1.23.0 - # via google-ai-generativelanguage - # via google-api-core -protobuf==4.25.3 - # via google-ai-generativelanguage - # via google-api-core - # via google-generativeai - # via googleapis-common-protos - # via grpcio-status - # via proto-plus -pyasn1==0.6.0 - # via pyasn1-modules - # via rsa -pyasn1-modules==0.4.0 - # via google-auth -pydantic==2.7.1 - # via anthropic - # via google-generativeai - # via groq - # via langchain - # via langchain-core - # via langsmith - # via openai - # via yahoo-search-py -pydantic-core==2.18.2 - # via pydantic -pyee==11.1.0 - # via playwright -pyparsing==3.1.2 - # via httplib2 -pytest==8.0.0 - # via pytest-mock -pytest-mock==3.14.0 -python-dateutil==2.9.0.post0 - # via botocore - # via pandas -python-dotenv==1.0.1 - # via scrapegraphai -pytz==2024.1 - # via pandas -pyyaml==6.0.1 - # via huggingface-hub - # via langchain - # via langchain-community - # via langchain-core -regex==2024.5.10 - # via tiktoken -requests==2.31.0 - # via free-proxy - # via google-api-core - # via huggingface-hub - # via langchain - # via langchain-community - # via langsmith - # via tiktoken -rsa==4.9 - # via google-auth -s3transfer==0.10.1 - # via boto3 -selectolax==0.3.21 - # via yahoo-search-py -six==1.16.0 - # via python-dateutil -sniffio==1.3.1 - # via anthropic - # via anyio - # via groq - # via httpx - # via openai -soupsieve==2.5 - # via beautifulsoup4 -sqlalchemy==2.0.30 - # via langchain - # via langchain-community -tenacity==8.3.0 - # via langchain - # via langchain-community - # via langchain-core -tiktoken==0.6.0 - # via langchain-openai - # via scrapegraphai -tokenizers==0.19.1 - # via anthropic -tomli==2.0.1 - # via pytest -tqdm==4.66.4 - # via google-generativeai - # via huggingface-hub - # via openai - # via scrapegraphai -typing-extensions==4.11.0 - # via anthropic - # via anyio - # via google-generativeai - # via groq - # via huggingface-hub - # via openai - # via pydantic - # via pydantic-core - # via pyee - # via sqlalchemy - # via typing-inspect -typing-inspect==0.9.0 - # via dataclasses-json -tzdata==2024.1 - # via pandas -uritemplate==4.1.1 - # via google-api-python-client -urllib3==1.26.18 - # via botocore - # via requests - # via yahoo-search-py -yahoo-search-py==0.3 - # via scrapegraphai -yarl==1.9.4 - # via aiohttp diff --git a/requirements.lock b/requirements.lock deleted file mode 100644 index c02d4522..00000000 --- a/requirements.lock +++ /dev/null @@ -1,300 +0,0 @@ -# generated by rye -# use `rye lock` or `rye sync` to update this lockfile -# -# last locked with the following flags: -# pre: false -# features: [] -# all-features: false -# with-sources: false - --e file:. -aiohttp==3.9.5 - # via langchain - # via langchain-community -aiosignal==1.3.1 - # via aiohttp -annotated-types==0.6.0 - # via pydantic -anthropic==0.25.9 - # via langchain-anthropic -anyio==4.3.0 - # via anthropic - # via groq - # via httpx - # via openai -async-timeout==4.0.3 - # via aiohttp - # via langchain -attrs==23.2.0 - # via aiohttp -beautifulsoup4==4.12.3 - # via google - # via scrapegraphai -boto3==1.34.105 - # via langchain-aws -botocore==1.34.105 - # via boto3 - # via s3transfer -cachetools==5.3.3 - # via google-auth -certifi==2024.2.2 - # via httpcore - # via httpx - # via requests -charset-normalizer==3.3.2 - # via requests -dataclasses-json==0.6.6 - # via langchain - # via langchain-community -defusedxml==0.7.1 - # via langchain-anthropic -distro==1.9.0 - # via anthropic - # via groq - # via openai -exceptiongroup==1.2.1 - # via anyio -faiss-cpu==1.8.0 - # via scrapegraphai -filelock==3.14.0 - # via huggingface-hub -free-proxy==1.1.1 - # via scrapegraphai -frozenlist==1.4.1 - # via aiohttp - # via aiosignal -fsspec==2024.3.1 - # via huggingface-hub -google==3.0.0 - # via scrapegraphai -google-ai-generativelanguage==0.6.3 - # via google-generativeai -google-api-core==2.19.0 - # via google-ai-generativelanguage - # via google-api-python-client - # via google-generativeai -google-api-python-client==2.129.0 - # via google-generativeai -google-auth==2.29.0 - # via google-ai-generativelanguage - # via google-api-core - # via google-api-python-client - # via google-auth-httplib2 - # via google-generativeai -google-auth-httplib2==0.2.0 - # via google-api-python-client -google-generativeai==0.5.3 - # via langchain-google-genai -googleapis-common-protos==1.63.0 - # via google-api-core - # via grpcio-status -graphviz==0.20.3 - # via scrapegraphai -greenlet==3.0.3 - # via playwright - # via sqlalchemy -groq==0.5.0 - # via langchain-groq -grpcio==1.63.0 - # via google-api-core - # via grpcio-status -grpcio-status==1.62.2 - # via google-api-core -h11==0.14.0 - # via httpcore -html2text==2024.2.26 - # via scrapegraphai -httpcore==1.0.5 - # via httpx -httplib2==0.22.0 - # via google-api-python-client - # via google-auth-httplib2 -httpx==0.27.0 - # via anthropic - # via groq - # via openai - # via yahoo-search-py -huggingface-hub==0.23.0 - # via tokenizers -idna==3.7 - # via anyio - # via httpx - # via requests - # via yarl -jmespath==1.0.1 - # via boto3 - # via botocore -jsonpatch==1.33 - # via langchain - # via langchain-core -jsonpointer==2.4 - # via jsonpatch -langchain==0.1.15 - # via scrapegraphai -langchain-anthropic==0.1.11 - # via scrapegraphai -langchain-aws==0.1.3 - # via scrapegraphai -langchain-community==0.0.38 - # via langchain -langchain-core==0.1.52 - # via langchain - # via langchain-anthropic - # via langchain-aws - # via langchain-community - # via langchain-google-genai - # via langchain-groq - # via langchain-openai - # via langchain-text-splitters -langchain-google-genai==1.0.3 - # via scrapegraphai -langchain-groq==0.1.3 - # via scrapegraphai -langchain-openai==0.1.6 - # via scrapegraphai -langchain-text-splitters==0.0.1 - # via langchain -langsmith==0.1.57 - # via langchain - # via langchain-community - # via langchain-core -lxml==5.2.2 - # via free-proxy -marshmallow==3.21.2 - # via dataclasses-json -minify-html==0.15.0 - # via scrapegraphai -multidict==6.0.5 - # via aiohttp - # via yarl -mypy-extensions==1.0.0 - # via typing-inspect -numpy==1.26.4 - # via faiss-cpu - # via langchain - # via langchain-aws - # via langchain-community - # via pandas -openai==1.30.1 - # via langchain-openai -orjson==3.10.3 - # via langsmith -packaging==23.2 - # via huggingface-hub - # via langchain-core - # via marshmallow -pandas==2.2.2 - # via scrapegraphai -playwright==1.43.0 - # via scrapegraphai -proto-plus==1.23.0 - # via google-ai-generativelanguage - # via google-api-core -protobuf==4.25.3 - # via google-ai-generativelanguage - # via google-api-core - # via google-generativeai - # via googleapis-common-protos - # via grpcio-status - # via proto-plus -pyasn1==0.6.0 - # via pyasn1-modules - # via rsa -pyasn1-modules==0.4.0 - # via google-auth -pydantic==2.7.1 - # via anthropic - # via google-generativeai - # via groq - # via langchain - # via langchain-core - # via langsmith - # via openai - # via yahoo-search-py -pydantic-core==2.18.2 - # via pydantic -pyee==11.1.0 - # via playwright -pyparsing==3.1.2 - # via httplib2 -python-dateutil==2.9.0.post0 - # via botocore - # via pandas -python-dotenv==1.0.1 - # via scrapegraphai -pytz==2024.1 - # via pandas -pyyaml==6.0.1 - # via huggingface-hub - # via langchain - # via langchain-community - # via langchain-core -regex==2024.5.10 - # via tiktoken -requests==2.31.0 - # via free-proxy - # via google-api-core - # via huggingface-hub - # via langchain - # via langchain-community - # via langsmith - # via tiktoken -rsa==4.9 - # via google-auth -s3transfer==0.10.1 - # via boto3 -selectolax==0.3.21 - # via yahoo-search-py -six==1.16.0 - # via python-dateutil -sniffio==1.3.1 - # via anthropic - # via anyio - # via groq - # via httpx - # via openai -soupsieve==2.5 - # via beautifulsoup4 -sqlalchemy==2.0.30 - # via langchain - # via langchain-community -tenacity==8.3.0 - # via langchain - # via langchain-community - # via langchain-core -tiktoken==0.6.0 - # via langchain-openai - # via scrapegraphai -tokenizers==0.19.1 - # via anthropic -tqdm==4.66.4 - # via google-generativeai - # via huggingface-hub - # via openai - # via scrapegraphai -typing-extensions==4.11.0 - # via anthropic - # via anyio - # via google-generativeai - # via groq - # via huggingface-hub - # via openai - # via pydantic - # via pydantic-core - # via pyee - # via sqlalchemy - # via typing-inspect -typing-inspect==0.9.0 - # via dataclasses-json -tzdata==2024.1 - # via pandas -uritemplate==4.1.1 - # via google-api-python-client -urllib3==1.26.18 - # via botocore - # via requests - # via yahoo-search-py -yahoo-search-py==0.3 - # via scrapegraphai -yarl==1.9.4 - # via aiohttp diff --git a/scrapegraphai/graphs/omni_search_graph.py b/scrapegraphai/graphs/omni_search_graph.py index 8dd5aba1..49f75c08 100644 --- a/scrapegraphai/graphs/omni_search_graph.py +++ b/scrapegraphai/graphs/omni_search_graph.py @@ -2,7 +2,7 @@ OmniSearchGraph Module """ -from copy import deepcopy +from copy import copy from .base_graph import BaseGraph from ..nodes import ( @@ -43,7 +43,7 @@ class OmniSearchGraph(AbstractGraph): def __init__(self, prompt: str, config: dict): self.max_results = config.get("max_results", 3) - self.copy_config = deepcopy(config) + self.copy_config = copy(config) super().__init__(prompt, config) diff --git a/scrapegraphai/graphs/search_graph.py b/scrapegraphai/graphs/search_graph.py index 58b7069c..6a46ab91 100644 --- a/scrapegraphai/graphs/search_graph.py +++ b/scrapegraphai/graphs/search_graph.py @@ -2,7 +2,7 @@ SearchGraph Module """ -from copy import deepcopy +from copy import copy from .base_graph import BaseGraph from ..nodes import ( @@ -42,7 +42,7 @@ class SearchGraph(AbstractGraph): def __init__(self, prompt: str, config: dict): self.max_results = config.get("max_results", 3) - self.copy_config = deepcopy(config) + self.copy_config = copy(config) super().__init__(prompt, config)