1
1
import { test as base } from '@playwright/test' ;
2
- import { z } from " zod" ;
2
+ import { z } from ' zod' ;
3
3
import { AutoConfig } from './types' ;
4
4
import { sessionManager , context } from './browser' ;
5
5
import { createReactAgent } from '@langchain/langgraph/prebuilt' ;
6
6
import { HumanMessage } from '@langchain/core/messages' ;
7
7
import { createLLMModel } from './llm' ;
8
8
import {
9
- browser_click ,
10
- browser_type ,
11
- browser_get_text ,
12
- browser_navigate ,
13
- browser_snapshot ,
14
- browser_hover ,
15
- browser_drag ,
16
- browser_select_option ,
17
- browser_take_screenshot ,
18
- browser_go_back ,
19
- browser_wait ,
20
- browser_press_key ,
21
- browser_save_pdf ,
22
- browser_choose_file ,
23
- browser_go_forward ,
24
- browser_assert ,
25
- browser_page_assert ,
9
+ browser_click ,
10
+ browser_type ,
11
+ browser_get_text ,
12
+ browser_navigate ,
13
+ browser_snapshot ,
14
+ browser_hover ,
15
+ browser_drag ,
16
+ browser_select_option ,
17
+ browser_take_screenshot ,
18
+ browser_go_back ,
19
+ browser_wait ,
20
+ browser_press_key ,
21
+ browser_save_pdf ,
22
+ browser_choose_file ,
23
+ browser_go_forward ,
24
+ browser_assert ,
25
+ browser_page_assert
26
26
} from './tools' ;
27
27
28
28
// Define response schema
29
29
const AutoResponseSchema = z . object ( {
30
- action : z . string ( ) . describe ( "The type of action performed (assert, click, type, etc)" ) ,
31
- error : z . string ( ) . describe ( "Error message if any, empty string if none" ) ,
32
- output : z . string ( ) . describe ( "Raw output from the action" )
30
+ action : z
31
+ . string ( )
32
+ . describe ( 'The type of action performed (assert, click, type, etc)' ) ,
33
+ error : z . string ( ) . describe ( 'Error message if any, empty string if none' ) ,
34
+ output : z . string ( ) . describe ( 'Raw output from the action' )
33
35
} ) ;
34
36
35
37
// Extend base test to automatically track page
36
38
export const test = base . extend ( {
37
- page : async ( { page } , use ) => {
38
- sessionManager . setPage ( page ) ;
39
- await use ( page ) ;
40
- } ,
39
+ page : async ( { page } , use ) => {
40
+ sessionManager . setPage ( page ) ;
41
+ await use ( page ) ;
42
+ }
41
43
} ) ;
42
44
43
45
// Initialize the LangChain agent with more detailed instructions
44
46
const initializeAgent = ( ) => {
45
- const model = createLLMModel ( ) ;
47
+ const model = createLLMModel ( ) ;
46
48
47
- const prompt = `You are a web automation assistant. When given a natural language instruction:
49
+ const prompt = `You are a web automation assistant. When given a natural language instruction:
48
50
- Always call the snapshot tool first to analyze the page structure and elements, so you can understand the context ad the elements available on the page to perform the requested action
49
51
- For "get" or "get text" instructions, use the getText tool to retrieve content
50
52
- For "click" instructions, use the click tool to interact with elements
@@ -70,98 +72,91 @@ const initializeAgent = () => {
70
72
"output": "<your output message>"
71
73
}` ;
72
74
73
- const agent = createReactAgent ( {
74
- llm : model ,
75
- tools : [
76
- browser_click ,
77
- browser_type ,
78
- browser_get_text ,
79
- browser_navigate ,
80
- browser_snapshot ,
81
- browser_hover ,
82
- browser_drag ,
83
- browser_select_option ,
84
- browser_take_screenshot ,
85
- browser_go_back ,
86
- browser_wait ,
87
- browser_press_key ,
88
- browser_save_pdf ,
89
- browser_choose_file ,
90
- browser_assert ,
91
- browser_go_forward ,
92
- browser_page_assert ,
93
- ] ,
94
- stateModifier : prompt ,
95
- responseFormat : {
96
- prompt : `Return a stringified JSON object with exactly these fields:
75
+ const agent = createReactAgent ( {
76
+ llm : model ,
77
+ tools : [
78
+ browser_click ,
79
+ browser_type ,
80
+ browser_get_text ,
81
+ browser_navigate ,
82
+ browser_snapshot ,
83
+ browser_hover ,
84
+ browser_drag ,
85
+ browser_select_option ,
86
+ browser_take_screenshot ,
87
+ browser_go_back ,
88
+ browser_wait ,
89
+ browser_press_key ,
90
+ browser_save_pdf ,
91
+ browser_choose_file ,
92
+ browser_assert ,
93
+ browser_go_forward ,
94
+ browser_page_assert
95
+ ] ,
96
+ stateModifier : prompt ,
97
+ responseFormat : {
98
+ prompt : `Return a stringified JSON object with exactly these fields:
97
99
{
98
100
"action": "<type of action performed>",
99
101
"error": "<error message or empty string>",
100
102
"output": "<your output message>"
101
103
}` ,
102
- schema : AutoResponseSchema
103
- }
104
- } ) ;
104
+ schema : AutoResponseSchema
105
+ }
106
+ } ) ;
105
107
106
- return { agent } ;
108
+ return { agent } ;
107
109
} ;
108
110
109
111
// Main auto function that processes instructions
110
112
export async function auto (
111
- instruction : string ,
112
- config ?: AutoConfig ,
113
+ instruction : string ,
114
+ config ?: AutoConfig
113
115
) : Promise < any > {
114
- console . log ( `[Auto] Processing instruction: "${ instruction } "` ) ;
116
+ console . log ( `[Auto] Processing instruction: "${ instruction } "` ) ;
115
117
116
- if ( config ?. page )
117
- {
118
- sessionManager . setPage ( config . page ) ;
119
- console . log ( `[Auto] Page set from config` ) ;
120
- } else
121
- {
122
- try
123
- {
124
- sessionManager . getPage ( ) ;
125
- } catch
126
- {
127
- // In standalone mode, create a new page
128
- console . log ( `[Auto] No existing page, creating new page` ) ;
129
- await context . createPage ( ) ;
130
- }
118
+ if ( config ?. page ) {
119
+ sessionManager . setPage ( config . page ) ;
120
+ console . log ( `[Auto] Page set from config` ) ;
121
+ } else {
122
+ try {
123
+ sessionManager . getPage ( ) ;
124
+ } catch {
125
+ // In standalone mode, create a new page
126
+ console . log ( `[Auto] No existing page, creating new page` ) ;
127
+ await context . createPage ( ) ;
131
128
}
129
+ }
132
130
133
- // Create and invoke the agent
134
- console . log ( `[Auto] Creating agent for instruction` ) ;
135
- const { agent } = initializeAgent ( ) ;
136
- const response = await agent . invoke ( {
137
- messages : [ new HumanMessage ( instruction ) ] ,
138
- } ) ;
139
- const result = response . structuredResponse ;
140
- // Process agent result
141
- try
142
- {
143
- console . log ( `[Auto] Agent response:` , result ) ;
144
-
145
- // Parse and validate the response
146
- const validatedResponse = AutoResponseSchema . parse ( result ) ;
131
+ // Create and invoke the agent
132
+ console . log ( `[Auto] Creating agent for instruction` ) ;
133
+ const { agent } = initializeAgent ( ) ;
134
+ const response = await agent . invoke ( {
135
+ messages : [ new HumanMessage ( instruction ) ]
136
+ } ) ;
137
+ const result = response . structuredResponse ;
138
+ // Process agent result
139
+ try {
140
+ console . log ( `[Auto] Agent response:` , result ) ;
147
141
148
- console . log ( `[Auto] Action: ${ validatedResponse . action } ` ) ;
149
- if ( validatedResponse . error )
150
- {
151
- console . log ( `[Auto] Error: ${ validatedResponse . error } ` ) ;
152
- throw {
153
- error : validatedResponse . error ,
154
- output : validatedResponse . output
155
- } ;
156
- }
142
+ // Parse and validate the response
143
+ const validatedResponse = AutoResponseSchema . parse ( result ) ;
157
144
158
- // Return the output or null if successful with no output
159
- return validatedResponse . output || null ;
160
- } catch ( error )
161
- {
162
- console . log ( `[Auto] Error processing response:` , error ) ;
163
- throw error ;
145
+ console . log ( `[Auto] Action: ${ validatedResponse . action } ` ) ;
146
+ if ( validatedResponse . error ) {
147
+ console . log ( `[Auto] Error: ${ validatedResponse . error } ` ) ;
148
+ throw {
149
+ error : validatedResponse . error ,
150
+ output : validatedResponse . output
151
+ } ;
164
152
}
153
+
154
+ // Return the output or null if successful with no output
155
+ return validatedResponse . output || null ;
156
+ } catch ( error ) {
157
+ console . log ( `[Auto] Error processing response:` , error ) ;
158
+ throw error ;
159
+ }
165
160
}
166
161
167
162
// Export everything needed for the package
0 commit comments