Skip to content

Commit bd5dd64

Browse files
authored
Merge pull request #90 from VinciGit00/fixing_bugs
fix: bug with fetch node
2 parents 3c77acb + 9cd5165 commit bd5dd64

File tree

5 files changed

+76
-14
lines changed

5 files changed

+76
-14
lines changed

examples/single_node/fetch_node.py

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
"""
2+
Example of custom graph using existing nodes
3+
"""
4+
5+
from scrapegraphai.nodes import FetchNode
6+
7+
# ************************************************
8+
# Define the node
9+
# ************************************************
10+
11+
12+
robots_node = FetchNode(
13+
input="url | local_dir",
14+
output=["doc"],
15+
)
16+
17+
# ************************************************
18+
# Test the node
19+
# ************************************************
20+
21+
state = {
22+
"url": "https://twitter.com/home"
23+
}
24+
25+
result = robots_node.execute(state)
26+
27+
print(result)

scrapegraphai/nodes/fetch_node.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,7 @@ def execute(self, state):
5959
6060
Raises:
6161
KeyError: If the 'url' key is not found in the state, indicating that the
62-
necessary information to perform the operation is missing.
62+
necessary information to perform the operation is missing.
6363
"""
6464
print(f"--- Executing {self.node_name} Node ---")
6565

@@ -78,7 +78,7 @@ def execute(self, state):
7878
})]
7979

8080
else:
81-
if self.node_config.get("endpoint") is not None:
81+
if self.node_config is not None and self.node_config.get("endpoint") is not None:
8282
loader = AsyncHtmlLoader(
8383
source, proxies={"http": self.node_config["endpoint"]})
8484
else:

tests/nodes/.env.example

Lines changed: 0 additions & 1 deletion
This file was deleted.

tests/nodes/fetch_node_test.py

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
"""
2+
Module for testinh robot_node
3+
"""
4+
import pytest
5+
from scrapegraphai.nodes import FetchNode
6+
7+
8+
@pytest.fixture
9+
def setup():
10+
"""
11+
setup
12+
"""
13+
# ************************************************
14+
# Define the node
15+
# ************************************************
16+
17+
robots_node = FetchNode(
18+
input="url | local_dir",
19+
output=["doc"],
20+
)
21+
22+
return robots_node
23+
24+
# ************************************************
25+
# Test the node
26+
# ************************************************
27+
28+
29+
def test_robots_node(setup):
30+
"""
31+
Run the tests
32+
"""
33+
state = {
34+
"url": "https://twitter.com/home"
35+
}
36+
37+
result = setup.execute(state)
38+
39+
assert result is not None
40+
41+
42+
# If you need to run this script directly
43+
if __name__ == "__main__":
44+
pytest.main()

tests/nodes/robot_node_test.py

Lines changed: 3 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,10 @@
11
"""
22
Module for testinh robot_node
33
"""
4-
import os
5-
from dotenv import load_dotenv
64
import pytest
7-
from scrapegraphai.models import OpenAI
5+
from scrapegraphai.models import Ollama
86
from scrapegraphai.nodes import RobotsNode
97

10-
# Load environment variables from .env file
11-
load_dotenv()
12-
138

149
@pytest.fixture
1510
def setup():
@@ -20,12 +15,9 @@ def setup():
2015
# Define the configuration for the graph
2116
# ************************************************
2217

23-
openai_key = os.getenv("OPENAI_APIKEY")
24-
2518
graph_config = {
2619
"llm": {
27-
"api_key": openai_key,
28-
"model": "gpt-3.5-turbo",
20+
"model": "ollama/llama3",
2921
"temperature": 0,
3022
"streaming": True
3123
},
@@ -35,7 +27,7 @@ def setup():
3527
# Define the node
3628
# ************************************************
3729

38-
llm_model = OpenAI(graph_config["llm"])
30+
llm_model = Ollama(graph_config["llm"])
3931

4032
robots_node = RobotsNode(
4133
input="url",

0 commit comments

Comments
 (0)