Skip to content

Commit b605fb1

Browse files
quanruyuyutaotao
andauthored
feat: add android-with-vitest-demo (#33)
* feat: add android-with-vitest-demo * feat: integrated with @midscene/android * feat: add more case * docs: add docs for android example * docs: update readme * docs: update readme --------- Co-authored-by: yutao <yutao.tao@bytedance.com>
1 parent c3d763f commit b605fb1

File tree

6 files changed

+329
-0
lines changed

6 files changed

+329
-0
lines changed

android-with-vitest-demo/.gitignore

+4
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
2+
# Midscene.js dump files
3+
midscene_run/report
4+
midscene_run/tmp

android-with-vitest-demo/README.md

+46
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
> Midscene x adb is still under development. You may use this demo if you want to have an early access.
2+
3+
# Android demo
4+
5+
This is a demo to show how to use bridge mode to control the page on your desktop Chrome.
6+
7+
## Steps
8+
9+
### Preparation
10+
11+
create `.env` file
12+
13+
```shell
14+
# replace by your gpt-4o api key
15+
OPENAI_API_KEY="YOUR_TOKEN"
16+
```
17+
18+
connect an Android device with [adb](https://developer.android.com/tools/adb)
19+
20+
Refer to this document if your want to use other models like Qwen: https://midscenejs.com/choose-a-model
21+
22+
### Install
23+
24+
install deps
25+
26+
```bash
27+
npm install
28+
```
29+
30+
### Run
31+
32+
case1:
33+
34+
```bash
35+
npm run test -- setting.test.ts
36+
```
37+
38+
or case2:
39+
40+
```
41+
npm run test -- todo.test.ts
42+
```
43+
44+
# Reference
45+
46+
https://midscenejs.com/api

android-with-vitest-demo/package.json

+18
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
{
2+
"name": "android-with-vitest-demo",
3+
"private": true,
4+
"version": "1.0.0",
5+
"main": "index.js",
6+
"type": "module",
7+
"scripts": {
8+
"test": "vitest --run"
9+
},
10+
"author": "",
11+
"license": "MIT",
12+
"devDependencies": {
13+
"@midscene/android": "beta",
14+
"@types/node": "^18.0.0",
15+
"dotenv": "^16.4.5",
16+
"vitest": "^2.1.8"
17+
}
18+
}
+174
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,174 @@
1+
import { exec } from 'node:child_process';
2+
import { promisify } from 'node:util';
3+
import { AndroidPage } from '@midscene/android';
4+
const execPromise = promisify(exec);
5+
6+
interface StartAppOptions {
7+
/**
8+
* The name of the application package
9+
*/
10+
pkg: string;
11+
/**
12+
* The name of the main application activity.
13+
* This or action is required in order to be able to launch an app.
14+
*/
15+
activity?: string;
16+
/**
17+
* The name of the intent action that will launch the required app.
18+
* This or activity is required in order to be able to launch an app.
19+
*/
20+
action?: string;
21+
/**
22+
* If this property is set to `true`
23+
* and the activity name does not start with '.' then the method
24+
* will try to add the missing dot and start the activity once more
25+
* if the first startup try fails.
26+
* `true` by default.
27+
*/
28+
retry?: boolean;
29+
/**
30+
* Set it to `true` in order to forcefully
31+
* stop the activity if it is already running.
32+
* `true` by default.
33+
*/
34+
stopApp?: boolean;
35+
/**
36+
* The name of the package to wait to on
37+
* startup (this only makes sense if this name is
38+
* different from the one, which is set as `pkg`)
39+
*/
40+
waitPkg?: string;
41+
/**
42+
* The name of the activity to wait to on
43+
* startup (this only makes sense if this name is different
44+
* from the one, which is set as `activity`)
45+
*/
46+
waitActivity?: string;
47+
/**
48+
* The number of milliseconds to wait until the
49+
* `waitActivity` is focused
50+
*/
51+
waitDuration?: number;
52+
/**
53+
* The number of the user profile to start
54+
* the given activity with. The default OS user profile (usually zero) is used
55+
* when this property is unset
56+
*/
57+
user?: string | number;
58+
/**
59+
* If `false` then adb won't wait
60+
* for the started activity to return the control.
61+
* `true` by default.
62+
*/
63+
waitForLaunch?: boolean;
64+
category?: string;
65+
flags?: string;
66+
optionalIntentArguments?: string;
67+
}
68+
69+
interface LaunchOptions {
70+
deviceId?: string;
71+
uri?: string;
72+
app?: StartAppOptions;
73+
}
74+
75+
/**
76+
* Get all connected Android device IDs
77+
* @returns List of device IDs
78+
* @throws Error when unable to retrieve device list
79+
*/
80+
export async function getConnectedDevices(): Promise<string[]> {
81+
try {
82+
const { stdout } = await execPromise('adb devices');
83+
const devices = stdout
84+
.split('\n')
85+
.slice(1) // Skip the first line "List of devices attached"
86+
.map((line) => {
87+
const [id, status] = line.split('\t');
88+
return { id, status };
89+
})
90+
.filter(({ id, status }) => id && status && status.trim() === 'device')
91+
.map(({ id }) => id);
92+
93+
return devices;
94+
} catch (error) {
95+
console.error('Failed to get device list:', error);
96+
throw new Error('Unable to get connected Android device list');
97+
}
98+
}
99+
100+
/**
101+
* Verify if the device is accessible
102+
* @param deviceId Device ID
103+
* @returns true if the device is accessible, false otherwise
104+
*/
105+
export async function isDeviceAccessible(deviceId: string): Promise<boolean> {
106+
try {
107+
await execPromise(`adb -s ${deviceId} shell echo "Device is ready"`);
108+
return true;
109+
} catch {
110+
return false;
111+
}
112+
}
113+
114+
/**
115+
* Launch Android page
116+
* @param opt Launch options
117+
* @returns AndroidPage instance
118+
* @throws Error when no available device is found
119+
*/
120+
export async function launchPage(opt: LaunchOptions): Promise<AndroidPage> {
121+
// If device ID is provided, use it directly
122+
let deviceId = opt.deviceId;
123+
124+
if (!deviceId) {
125+
// Get all connected devices
126+
const devices = await getConnectedDevices();
127+
128+
if (devices.length === 0) {
129+
throw new Error('No available Android devices found');
130+
}
131+
132+
if (devices.length > 1) {
133+
console.warn(
134+
`Multiple devices detected: ${devices.join(', ')}. Using the first device: ${devices[0]}`,
135+
);
136+
}
137+
138+
// Use the first available device
139+
deviceId = devices[0];
140+
}
141+
142+
// Verify if the device is accessible
143+
const isAccessible = await isDeviceAccessible(deviceId);
144+
if (!isAccessible) {
145+
throw new Error(
146+
`Device ${deviceId} is not accessible, please check the connection status`,
147+
);
148+
}
149+
150+
const androidPage = new AndroidPage({
151+
deviceId,
152+
});
153+
154+
const adb = await androidPage.getAdb();
155+
156+
// handle URI (if provided), support app page and web page
157+
if (opt.uri) {
158+
try {
159+
await adb.startUri(opt.uri);
160+
} catch (error) {
161+
console.error(`Error starting URI: ${error}`);
162+
}
163+
}
164+
165+
if (opt.app) {
166+
try {
167+
await adb.startApp(opt.app);
168+
} catch (error) {
169+
console.error(`Error starting app: ${error}`);
170+
}
171+
}
172+
173+
return androidPage;
174+
}
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
import { AndroidAgent } from '@midscene/android';
2+
import { describe, it, vi } from 'vitest';
3+
import { launchPage } from './helper';
4+
import 'dotenv/config'; // read environment variables from .env file
5+
6+
vi.setConfig({
7+
testTimeout: 90 * 1000,
8+
});
9+
10+
const DEVICE_ID = process.env.ANDROID_DEVICE_ID;
11+
12+
describe(
13+
'android integration',
14+
async () => {
15+
await it('Android settings page demo for scroll', async () => {
16+
const page = await launchPage({
17+
deviceId: DEVICE_ID,
18+
app: {
19+
pkg: 'com.android.settings',
20+
activity: '.Settings',
21+
},
22+
});
23+
const agent = new AndroidAgent(page);
24+
25+
await agent.aiAction('scroll list to bottom');
26+
await agent.aiAction('open "More settings"');
27+
await agent.aiAction('scroll list to bottom');
28+
await agent.aiAction('scroll list to top');
29+
await agent.aiAction('swipe down one screen');
30+
await agent.aiAction('swipe up one screen');
31+
});
32+
},
33+
360 * 1000,
34+
);
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
import { AndroidAgent } from '@midscene/android';
2+
import { beforeAll, describe, expect, it, vi } from 'vitest';
3+
import { launchPage } from './helper';
4+
import 'dotenv/config'; // read environment variables from .env file
5+
6+
vi.setConfig({
7+
testTimeout: 240 * 1000,
8+
});
9+
10+
const pageUrl = 'https://todomvc.com/examples/react/dist/';
11+
const DEVICE_ID = process.env.ANDROID_DEVICE_ID;
12+
13+
describe('Test todo list', () => {
14+
let agent: AndroidAgent;
15+
16+
beforeAll(async () => {
17+
agent = new AndroidAgent(
18+
await launchPage({ deviceId: DEVICE_ID, uri: pageUrl }),
19+
);
20+
});
21+
22+
it(
23+
'ai todo',
24+
async () => {
25+
await agent.aiAction(
26+
"type 'Study JS today' in the task box input and press the Enter key",
27+
);
28+
await agent.aiAction(
29+
"type 'Study Rust tomorrow' in the task box input and press the Enter key",
30+
);
31+
await agent.aiAction(
32+
"type 'Study AI the day after tomorrow' in the task box input and press the Enter key",
33+
);
34+
await agent.aiAction(
35+
'move the mouse to the second item in the task list and click the delete button on the right of the second task',
36+
);
37+
await agent.aiAction(
38+
'click the check button on the left of the second task',
39+
);
40+
await agent.aiAction(
41+
"click the 'completed' status button below the task list",
42+
);
43+
44+
const list = await agent.aiQuery('string[], the complete task list');
45+
expect(list.length).toEqual(1);
46+
47+
await agent.aiAssert(
48+
'Near the bottom of the list, there is a tip shows "1 item left".',
49+
);
50+
},
51+
720 * 1000,
52+
);
53+
});

0 commit comments

Comments
 (0)