Merge pull request #292 from VinciGit00/refactoring

Refactoring

pyproject.toml

@@ -67,7 +67,7 @@ classifiers = [
     "Programming Language :: Python :: 3",
     "Operating System :: OS Independent",
 ]
-requires-python = ">= 3.9"
+requires-python = ">=3.9,<3.12"
 
 [build-system]
 requires = ["hatchling"]

requirements-dev.lock

@@ -45,10 +45,6 @@ certifi==2024.2.2
     # via requests
 charset-normalizer==3.3.2
     # via requests
-colorama==0.4.6
-    # via ipython
-    # via pytest
-    # via tqdm
 dataclasses-json==0.6.6
     # via langchain
     # via langchain-community
@@ -104,7 +100,6 @@ graphviz==0.20.3
     # via scrapegraphai
 greenlet==3.0.3
     # via playwright
-    # via sqlalchemy
 groq==0.5.0
     # via langchain-groq
 grpcio==1.63.0
@@ -217,8 +212,11 @@ pandas==2.2.2
     # via scrapegraphai
 parso==0.8.4
     # via jedi
+pexpect==4.9.0
+    # via ipython
 playwright==1.43.0
     # via scrapegraphai
+    # via undetected-playwright
 pluggy==1.5.0
     # via pytest
 prompt-toolkit==3.0.43
@@ -233,6 +231,8 @@ protobuf==4.25.3
     # via googleapis-common-protos
     # via grpcio-status
     # via proto-plus
+ptyprocess==0.7.0
+    # via pexpect
 pure-eval==0.2.2
     # via stack-data
 pyasn1==0.6.0
@@ -342,6 +342,8 @@ typing-inspect==0.9.0
     # via dataclasses-json
 tzdata==2024.1
     # via pandas
+undetected-playwright==0.3.0
+    # via scrapegraphai
 uritemplate==4.1.1
     # via google-api-python-client
 urllib3==2.2.1

requirements.lock

@@ -45,9 +45,6 @@ certifi==2024.2.2
     # via requests
 charset-normalizer==3.3.2
     # via requests
-colorama==0.4.6
-    # via ipython
-    # via tqdm
 dataclasses-json==0.6.6
     # via langchain
     # via langchain-community
@@ -102,7 +99,6 @@ graphviz==0.20.3
     # via scrapegraphai
 greenlet==3.0.3
     # via playwright
-    # via sqlalchemy
 groq==0.5.0
     # via langchain-groq
 grpcio==1.63.0
@@ -212,8 +208,11 @@ pandas==2.2.2
     # via scrapegraphai
 parso==0.8.4
     # via jedi
+pexpect==4.9.0
+    # via ipython
 playwright==1.43.0
     # via scrapegraphai
+    # via undetected-playwright
 prompt-toolkit==3.0.43
     # via ipython
 proto-plus==1.23.0
@@ -226,6 +225,8 @@ protobuf==4.25.3
     # via googleapis-common-protos
     # via grpcio-status
     # via proto-plus
+ptyprocess==0.7.0
+    # via pexpect
 pure-eval==0.2.2
     # via stack-data
 pyasn1==0.6.0
@@ -330,6 +331,8 @@ typing-inspect==0.9.0
     # via dataclasses-json
 tzdata==2024.1
     # via pandas
+undetected-playwright==0.3.0
+    # via scrapegraphai
 uritemplate==4.1.1
     # via google-api-python-client
 urllib3==2.2.1

scrapegraphai/graphs/abstract_graph.py

@@ -354,4 +354,4 @@ def run(self) -> str:
         """
         Abstract method to execute the graph and return the result.
         """
-        pass
+        pass

scrapegraphai/graphs/pdf_scraper_graph.py

@@ -114,4 +114,4 @@ def run(self) -> str:
         inputs = {"user_prompt": self.prompt, self.input_key: self.source}
         self.final_state, self.execution_info = self.graph.execute(inputs)
 
-        return self.final_state.get("answer", "No answer found.")
+        return self.final_state.get("answer", "No answer found.")

scrapegraphai/helpers/models_tokens.py

@@ -23,7 +23,10 @@
     "azure": {
         "gpt-3.5-turbo": 4096,
         "gpt-4": 8192,
-        "gpt-4-32k": 32768
+        "gpt-4-0613": 8192,
+        "gpt-4-32k": 32768,
+        "gpt-4-32k-0613": 32768,
+        "gpt-4o": 128000,
     },
     "gemini": {
         "gemini-pro": 128000,
@@ -141,7 +144,8 @@
         "cognitivecomputations/dolphin-2.5-mixtral-8x7b": 32768,
         "TheBloke/dolphin-2.7-mixtral-8x7b-GGUF": 32768,
         "deepseek-ai/DeepSeek-V2": 131072,
-        "deepseek-ai/DeepSeek-V2-Chat": 131072
+        "deepseek-ai/DeepSeek-V2-Chat": 131072,
+        "claude-3-haiku": 200000
     },
     "deepseek": {
         "deepseek-chat": 32768,

scrapegraphai/nodes/generate_answer_pdf_node.py

@@ -49,7 +49,7 @@ def __init__(self, input: str, output: List[str], node_config: Optional[dict] =
             node_name (str): name of the node
         """
         super().__init__(node_name, "node", input, output, 2, node_config)
-        self.llm_model = node_config["llm"]
+        self.llm_model = node_config["llm_model"]
         self.verbose = False if node_config is None else node_config.get(
             "verbose", False)