Skip to content

Commit e26fc45

Browse files
JonasBaJonasBa
authored
ref(trace): output a flattened list of tokens with injected AND and proper parenthesis as tokens (#69591)
This will enable us to construct an ast based on the precedence order. From here, all we need to do is convert to infix/postfix and evaluate the expression
1 parent f8d3a1e commit e26fc45

File tree

2 files changed

+292
-0
lines changed

2 files changed

+292
-0
lines changed

‎static/app/components/searchSyntax/evaluator.spec.tsx

Lines changed: 200 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,200 @@
1+
import {
2+
insertImplicitAND,
3+
type ProcessedTokenResult,
4+
toFlattened,
5+
} from 'sentry/components/searchSyntax/evaluator';
6+
import {
7+
parseSearch,
8+
Token,
9+
type TokenResult,
10+
} from 'sentry/components/searchSyntax/parser';
11+
12+
const tokensToString = (tokens: ProcessedTokenResult[]): string => {
13+
let str = '';
14+
15+
for (const token of tokens) {
16+
let concatstr;
17+
switch (token.type) {
18+
case Token.FREE_TEXT:
19+
concatstr = token.text;
20+
break;
21+
case Token.SPACES:
22+
concatstr = 'space';
23+
break;
24+
case Token.VALUE_DURATION:
25+
case Token.VALUE_BOOLEAN:
26+
case Token.VALUE_NUMBER:
27+
case Token.VALUE_SIZE:
28+
case Token.VALUE_PERCENTAGE:
29+
case Token.VALUE_TEXT:
30+
case Token.VALUE_ISO_8601_DATE:
31+
case Token.VALUE_RELATIVE_DATE:
32+
concatstr = token.value;
33+
break;
34+
case Token.LOGIC_GROUP:
35+
case Token.LOGIC_BOOLEAN:
36+
concatstr = token.text;
37+
break;
38+
case Token.KEY_SIMPLE:
39+
concatstr = token.text + ':';
40+
break;
41+
case Token.VALUE_NUMBER_LIST:
42+
case Token.VALUE_TEXT_LIST:
43+
concatstr = token.text;
44+
break;
45+
case Token.KEY_EXPLICIT_TAG:
46+
concatstr = token.key;
47+
break;
48+
case 'L_PAREN': {
49+
concatstr = '(';
50+
break;
51+
}
52+
case 'R_PAREN': {
53+
concatstr = ')';
54+
break;
55+
}
56+
default: {
57+
concatstr = token.text;
58+
break;
59+
}
60+
}
61+
62+
// The parsing logic text() captures leading/trailing spaces in some cases.
63+
// We'll just trim them so the tests are easier to read.
64+
str += concatstr.trim();
65+
str += concatstr && tokens.indexOf(token) !== tokens.length - 1 ? ' ' : '';
66+
}
67+
68+
return str;
69+
};
70+
71+
function assertTokens(
72+
tokens: TokenResult<Token>[] | null
73+
): asserts tokens is TokenResult<Token>[] {
74+
if (tokens === null) {
75+
throw new Error('Expected tokens to be an array');
76+
}
77+
}
78+
79+
describe('Search Syntax Evaluator', () => {
80+
describe('flatten tree', () => {
81+
it('flattens simple expressions', () => {
82+
const tokens = parseSearch('is:unresolved duration:>1h');
83+
assertTokens(tokens);
84+
const flattened = toFlattened(tokens);
85+
expect(flattened).toHaveLength(2);
86+
expect(tokensToString(flattened)).toBe('is:unresolved duration:>1h');
87+
});
88+
it('handles filters', () => {
89+
const tokens = parseSearch('has:unresolved duration:[1,2,3]');
90+
assertTokens(tokens);
91+
const flattened = toFlattened(tokens);
92+
expect(flattened).toHaveLength(2);
93+
expect(tokensToString(flattened)).toBe('has:unresolved duration:[1,2,3]');
94+
});
95+
it('handles free text', () => {
96+
const tokens = parseSearch('hello world');
97+
assertTokens(tokens);
98+
const flattened = toFlattened(tokens);
99+
expect(flattened).toHaveLength(1);
100+
expect(tokensToString(flattened)).toBe('hello world');
101+
});
102+
it('handles logical booleans', () => {
103+
const tokens = parseSearch('hello AND world');
104+
assertTokens(tokens);
105+
const flattened = toFlattened(tokens);
106+
expect(flattened).toHaveLength(3);
107+
expect(tokensToString(flattened)).toBe('hello AND world');
108+
});
109+
it('handles logical groups', () => {
110+
const tokens = parseSearch('is:unresolved AND (is:dead OR is:alive)');
111+
assertTokens(tokens);
112+
const flattened = toFlattened(tokens);
113+
expect(flattened).toHaveLength(7);
114+
expect(tokensToString(flattened)).toBe('is:unresolved AND ( is:dead OR is:alive )');
115+
});
116+
});
117+
118+
describe('injects implicit AND', () => {
119+
describe('boolean operators', () => {
120+
it('implicit AND', () => {
121+
const tokens = toFlattened(parseSearch('is:unresolved duration:>1h')!);
122+
const withImplicitAND = insertImplicitAND(tokens);
123+
expect(tokensToString(withImplicitAND)).toBe('is:unresolved AND duration:>1h');
124+
});
125+
126+
it('explicit AND', () => {
127+
const tokens = toFlattened(parseSearch('is:unresolved AND duration:>1h')!);
128+
const withImplicitAND = insertImplicitAND(tokens);
129+
expect(tokensToString(withImplicitAND)).toBe('is:unresolved AND duration:>1h');
130+
});
131+
132+
it('multiple implicit AND', () => {
133+
const tokens = toFlattened(
134+
parseSearch('is:unresolved duration:>1h duration:<1m')!
135+
);
136+
const withImplicitAND = insertImplicitAND(tokens);
137+
expect(tokensToString(withImplicitAND)).toBe(
138+
'is:unresolved AND duration:>1h AND duration:<1m'
139+
);
140+
});
141+
142+
it('explicit OR', () => {
143+
const tokens = toFlattened(parseSearch('is:unresolved OR duration:>1h')!);
144+
const withImplicitAND = insertImplicitAND(tokens);
145+
expect(tokensToString(withImplicitAND)).toBe('is:unresolved OR duration:>1h');
146+
});
147+
148+
it('multiple explicit OR', () => {
149+
const tokens = toFlattened(
150+
parseSearch('is:unresolved OR duration:>1h OR duration:<1h')!
151+
);
152+
const withImplicitAND = insertImplicitAND(tokens);
153+
expect(tokensToString(withImplicitAND)).toBe(
154+
'is:unresolved OR duration:>1h OR duration:<1h'
155+
);
156+
});
157+
158+
it('with logical groups', () => {
159+
const tokens = toFlattened(parseSearch('is:unresolved (duration:>1h)')!);
160+
const withImplicitAND = insertImplicitAND(tokens);
161+
expect(tokensToString(withImplicitAND)).toBe(
162+
'is:unresolved AND ( duration:>1h )'
163+
);
164+
});
165+
});
166+
167+
describe('logical groups', () => {
168+
it('explicit OR', () => {
169+
const tokens = toFlattened(parseSearch('is:unresolved OR ( duration:>1h )')!);
170+
const withImplicitAND = insertImplicitAND(tokens);
171+
expect(tokensToString(withImplicitAND)).toBe('is:unresolved OR ( duration:>1h )');
172+
});
173+
it('explicit AND', () => {
174+
const tokens = toFlattened(parseSearch('is:unresolved AND ( duration:>1h )')!);
175+
expect(tokensToString(tokens)).toBe('is:unresolved AND ( duration:>1h )');
176+
});
177+
});
178+
179+
describe('complex expressions', () => {
180+
it('handles complex expressions', () => {
181+
const tokens = toFlattened(
182+
parseSearch('is:unresolved AND ( duration:>1h OR duration:<1h )')!
183+
);
184+
expect(tokensToString(tokens)).toBe(
185+
'is:unresolved AND ( duration:>1h OR duration:<1h )'
186+
);
187+
});
188+
189+
it('handles complex expressions with implicit AND', () => {
190+
const tokens = toFlattened(
191+
parseSearch('is:unresolved ( duration:>1h OR ( duration:<1h duration:1m ) )')!
192+
);
193+
const withImplicitAND = insertImplicitAND(tokens);
194+
expect(tokensToString(withImplicitAND)).toBe(
195+
'is:unresolved AND ( duration:>1h OR ( duration:<1h AND duration:1m ) )'
196+
);
197+
});
198+
});
199+
});
200+
});

‎static/app/components/searchSyntax/evaluator.tsx

Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,92 @@
1+
// To evaluate a result of the search syntax, we flatten the AST,
2+
// transform it to postfix notation which gets rid of parenthesis and tokens
3+
// that do not hold any value as they cannot be evaluated and then evaluate
4+
// the postfix notation.
5+
6+
import {
7+
BooleanOperator,
8+
Token,
9+
type TokenResult,
10+
} from 'sentry/components/searchSyntax/parser';
11+
12+
export type ProcessedTokenResult =
13+
| TokenResult<Token>
14+
| {type: 'L_PAREN'}
15+
| {type: 'R_PAREN'};
16+
17+
export function toFlattened(tokens: TokenResult<Token>[]): ProcessedTokenResult[] {
18+
const flattened_result: ProcessedTokenResult[] = [];
19+
20+
function flatten(token: TokenResult<Token>): void {
21+
switch (token.type) {
22+
case Token.SPACES:
23+
case Token.VALUE_BOOLEAN:
24+
case Token.VALUE_DURATION:
25+
case Token.VALUE_ISO_8601_DATE:
26+
case Token.VALUE_SIZE:
27+
case Token.VALUE_NUMBER_LIST:
28+
case Token.VALUE_NUMBER:
29+
case Token.VALUE_TEXT:
30+
case Token.VALUE_TEXT_LIST:
31+
case Token.VALUE_RELATIVE_DATE:
32+
case Token.VALUE_PERCENTAGE:
33+
case Token.KEY_SIMPLE:
34+
return;
35+
case Token.LOGIC_GROUP:
36+
flattened_result.push({type: 'L_PAREN'});
37+
for (const child of token.inner) {
38+
// Logic groups are wrapped in parenthesis,
39+
// but those parenthesis are not actual tokens returned by the parser
40+
flatten(child);
41+
}
42+
flattened_result.push({type: 'R_PAREN'});
43+
break;
44+
case Token.LOGIC_BOOLEAN:
45+
flattened_result.push(token);
46+
break;
47+
default:
48+
flattened_result.push(token);
49+
break;
50+
}
51+
}
52+
53+
for (let i = 0; i < tokens.length; i++) {
54+
flatten(tokens[i]);
55+
}
56+
57+
return flattened_result;
58+
}
59+
60+
// At this point we have a flat list of groups that we can evaluate, however since the syntax allows
61+
// implicit ANDs, we should still insert those as it will make constructing a valid AST easier
62+
export function insertImplicitAND(
63+
tokens: ProcessedTokenResult[]
64+
): ProcessedTokenResult[] {
65+
const with_implicit_and: ProcessedTokenResult[] = [];
66+
67+
const AND = {
68+
type: Token.LOGIC_BOOLEAN,
69+
value: BooleanOperator.AND,
70+
text: 'AND',
71+
location: null as unknown as PEG.LocationRange,
72+
invalid: null,
73+
} as TokenResult<Token>;
74+
75+
for (let i = 0; i < tokens.length; i++) {
76+
const next = tokens[i + 1];
77+
with_implicit_and.push(tokens[i]);
78+
79+
// If current is not a logic boolean and next is not a logic boolean, insert an implicit AND.
80+
if (
81+
next &&
82+
next.type !== Token.LOGIC_BOOLEAN &&
83+
tokens[i].type !== Token.LOGIC_BOOLEAN &&
84+
tokens[i].type !== 'L_PAREN' &&
85+
next.type !== 'R_PAREN'
86+
) {
87+
with_implicit_and.push(AND);
88+
}
89+
}
90+
91+
return with_implicit_and;
92+
}

0 commit comments

Comments
 (0)