File tree Expand file tree Collapse file tree 5 files changed +76
-14
lines changed Expand file tree Collapse file tree 5 files changed +76
-14
lines changed Original file line number Diff line number Diff line change
1
+ """
2
+ Example of custom graph using existing nodes
3
+ """
4
+
5
+ from scrapegraphai .nodes import FetchNode
6
+
7
+ # ************************************************
8
+ # Define the node
9
+ # ************************************************
10
+
11
+
12
+ robots_node = FetchNode (
13
+ input = "url | local_dir" ,
14
+ output = ["doc" ],
15
+ )
16
+
17
+ # ************************************************
18
+ # Test the node
19
+ # ************************************************
20
+
21
+ state = {
22
+ "url" : "https://twitter.com/home"
23
+ }
24
+
25
+ result = robots_node .execute (state )
26
+
27
+ print (result )
Original file line number Diff line number Diff line change @@ -59,7 +59,7 @@ def execute(self, state):
59
59
60
60
Raises:
61
61
KeyError: If the 'url' key is not found in the state, indicating that the
62
- necessary information to perform the operation is missing.
62
+ necessary information to perform the operation is missing.
63
63
"""
64
64
print (f"--- Executing { self .node_name } Node ---" )
65
65
@@ -78,7 +78,7 @@ def execute(self, state):
78
78
})]
79
79
80
80
else :
81
- if self .node_config .get ("endpoint" ) is not None :
81
+ if self .node_config is not None and self . node_config .get ("endpoint" ) is not None :
82
82
loader = AsyncHtmlLoader (
83
83
source , proxies = {"http" : self .node_config ["endpoint" ]})
84
84
else :
Load Diff This file was deleted.
Original file line number Diff line number Diff line change
1
+ """
2
+ Module for testinh robot_node
3
+ """
4
+ import pytest
5
+ from scrapegraphai .nodes import FetchNode
6
+
7
+
8
+ @pytest .fixture
9
+ def setup ():
10
+ """
11
+ setup
12
+ """
13
+ # ************************************************
14
+ # Define the node
15
+ # ************************************************
16
+
17
+ robots_node = FetchNode (
18
+ input = "url | local_dir" ,
19
+ output = ["doc" ],
20
+ )
21
+
22
+ return robots_node
23
+
24
+ # ************************************************
25
+ # Test the node
26
+ # ************************************************
27
+
28
+
29
+ def test_robots_node (setup ):
30
+ """
31
+ Run the tests
32
+ """
33
+ state = {
34
+ "url" : "https://twitter.com/home"
35
+ }
36
+
37
+ result = setup .execute (state )
38
+
39
+ assert result is not None
40
+
41
+
42
+ # If you need to run this script directly
43
+ if __name__ == "__main__" :
44
+ pytest .main ()
Original file line number Diff line number Diff line change 1
1
"""
2
2
Module for testinh robot_node
3
3
"""
4
- import os
5
- from dotenv import load_dotenv
6
4
import pytest
7
- from scrapegraphai .models import OpenAI
5
+ from scrapegraphai .models import Ollama
8
6
from scrapegraphai .nodes import RobotsNode
9
7
10
- # Load environment variables from .env file
11
- load_dotenv ()
12
-
13
8
14
9
@pytest .fixture
15
10
def setup ():
@@ -20,12 +15,9 @@ def setup():
20
15
# Define the configuration for the graph
21
16
# ************************************************
22
17
23
- openai_key = os .getenv ("OPENAI_APIKEY" )
24
-
25
18
graph_config = {
26
19
"llm" : {
27
- "api_key" : openai_key ,
28
- "model" : "gpt-3.5-turbo" ,
20
+ "model" : "ollama/llama3" ,
29
21
"temperature" : 0 ,
30
22
"streaming" : True
31
23
},
@@ -35,7 +27,7 @@ def setup():
35
27
# Define the node
36
28
# ************************************************
37
29
38
- llm_model = OpenAI (graph_config ["llm" ])
30
+ llm_model = Ollama (graph_config ["llm" ])
39
31
40
32
robots_node = RobotsNode (
41
33
input = "url" ,
You can’t perform that action at this time.
0 commit comments