Skip to content

Commit 0a1a1d5

Browse files
zaidjan-devrevzaidjan-devrev
authored
perf(in-filter): Update in filter to use any operator for performance improvements (#165)
* added any filter * added not in operator update * smaller AST * recd '/Users/zaidjan/Documents/Projects/meerkat' * updated comments * updated versions * added number and string based filtering with string split * common delimiter * added combined test cases * added combined test cases * elongate the delimiter
1 parent 197b5eb commit 0a1a1d5

File tree

9 files changed

+1122
-243
lines changed

9 files changed

+1122
-243
lines changed

‎meerkat-browser/package.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{
22
"name": "@devrev/meerkat-browser",
3-
"version": "0.0.104",
3+
"version": "0.0.105",
44
"dependencies": {
55
"tslib": "^2.3.0",
66
"@devrev/meerkat-core": "*",

‎meerkat-core/package.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{
22
"name": "@devrev/meerkat-core",
3-
"version": "0.0.104",
3+
"version": "0.0.105",
44
"dependencies": {
55
"tslib": "^2.3.0"
66
},

‎meerkat-core/src/cube-filter-transformer/in/in.spec.ts‎

Lines changed: 205 additions & 104 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ import { inTransform } from './in';
44
describe('In transforms Tests', () => {
55
it('Should throw error if values are undefined', () => {
66
expect(() =>
7-
inTransform({
7+
inTransform({
88
member: 'country',
99
operator: 'contains',
1010
memberInfo: {
@@ -16,130 +16,231 @@ describe('In transforms Tests', () => {
1616
).toThrow();
1717
});
1818

19-
it('Should return the correct value for string member', () => {
20-
const expectedOutput = {
21-
"alias": "",
22-
"children": [
23-
{
24-
"alias": "",
25-
"class": "COLUMN_REF",
26-
"column_names": [
27-
"country",
28-
],
29-
"type": "COLUMN_REF",
30-
},
31-
{
32-
"alias": "",
33-
"class": "CONSTANT",
34-
"type": "VALUE_CONSTANT",
35-
"value": {
36-
"is_null": false,
37-
"type": {
38-
"id": "VARCHAR",
39-
"type_info": null,
40-
},
41-
"value": "US",
42-
},
43-
},
44-
],
45-
"class": "OPERATOR",
46-
"type": "COMPARE_IN",
47-
};
48-
expect(
49-
inTransform({
50-
member: 'country',
51-
operator: 'contains',
52-
values: ['US'],
53-
memberInfo: {
54-
name: 'country',
55-
sql: 'table.country',
56-
type: 'string',
57-
},
58-
})
59-
).toEqual(expectedOutput);
19+
it('Should return optimized string_split approach for string type', () => {
20+
const result = inTransform({
21+
member: 'country',
22+
operator: 'in',
23+
values: ['US', 'Canada', 'Mexico'],
24+
memberInfo: {
25+
name: 'country',
26+
sql: 'table.country',
27+
type: 'string',
28+
},
29+
});
30+
31+
// Check it returns a subquery structure with string_split
32+
expect(result).toHaveProperty('class', 'SUBQUERY');
33+
expect(result).toHaveProperty('type', 'SUBQUERY');
34+
expect(result).toHaveProperty('subquery_type', 'ANY');
35+
36+
// Verify it's using string_split
37+
const selectList = (result as any).subquery.node.select_list[0];
38+
expect(selectList.function_name).toBe('unnest');
39+
expect(selectList.children[0].function_name).toBe('string_split');
40+
41+
// Verify no CAST for strings
42+
expect(selectList.type).toBe('FUNCTION');
6043
});
6144

62-
it('Should return the correct value for string_array member', () => {
45+
it('Should return optimized string_split approach with CAST for number type', () => {
46+
const result = inTransform({
47+
member: 'order_id',
48+
operator: 'in',
49+
values: [1, 2, 3],
50+
memberInfo: {
51+
name: 'order_id',
52+
sql: 'table.order_id',
53+
type: 'number',
54+
},
55+
});
56+
57+
// Check it returns a subquery structure
58+
expect(result).toHaveProperty('class', 'SUBQUERY');
59+
expect(result).toHaveProperty('type', 'SUBQUERY');
60+
expect(result).toHaveProperty('subquery_type', 'ANY');
61+
62+
// Verify it's using string_split with CAST
63+
const selectList = (result as any).subquery.node.select_list[0];
64+
expect(selectList.type).toBe('OPERATOR_CAST');
65+
expect(selectList.cast_type.id).toBe('DOUBLE');
66+
expect(selectList.child.function_name).toBe('unnest');
67+
expect(selectList.child.children[0].function_name).toBe('string_split');
68+
});
69+
70+
it('Should return standard ARRAY_CONSTRUCTOR for string_array type', () => {
6371
const output = inTransform({
6472
member: 'country',
65-
operator: 'contains',
73+
operator: 'in',
6674
values: ['US', 'Germany', 'Israel'],
6775
memberInfo: {
6876
name: 'country',
6977
sql: 'table.country',
7078
type: 'string_array',
7179
},
7280
}) as ConjunctionExpression;
73-
expect(output).toEqual( {
74-
"alias": "",
75-
"catalog": "",
76-
"children": [
77-
{
78-
"alias": "",
79-
"class": "COLUMN_REF",
80-
"column_names": [
81-
"country",
82-
],
83-
"type": "COLUMN_REF",
81+
82+
// For array types, should use && operator with ARRAY_CONSTRUCTOR
83+
expect(output.function_name).toBe('&&');
84+
expect(output.children[1].type).toBe('ARRAY_CONSTRUCTOR');
85+
expect(output.children[1].children.length).toBe(3);
86+
});
87+
88+
it('Should return standard COMPARE_IN for other types (default case)', () => {
89+
const output = inTransform({
90+
member: 'some_field',
91+
operator: 'in',
92+
values: ['val1', 'val2'],
93+
memberInfo: {
94+
name: 'some_field',
95+
sql: 'table.some_field',
96+
type: 'time' as any, // Unknown type to trigger default case
97+
},
98+
});
99+
100+
// Default case should use COMPARE_IN
101+
expect(output).toHaveProperty('type', 'COMPARE_IN');
102+
expect(output).toHaveProperty('class', 'OPERATOR');
103+
expect((output as any).children.length).toBe(3); // column + 2 values
104+
});
105+
106+
it('Should handle large value lists efficiently with string_split', () => {
107+
const largeValueList = Array.from({ length: 1000 }, (_, i) => `value${i}`);
108+
const result = inTransform({
109+
member: 'country',
110+
operator: 'in',
111+
values: largeValueList,
112+
memberInfo: {
113+
name: 'country',
114+
sql: 'table.country',
115+
type: 'string',
116+
},
117+
});
118+
119+
// Should still use subquery approach
120+
expect(result).toHaveProperty('class', 'SUBQUERY');
121+
122+
// Verify only 2 VALUE_CONSTANT nodes (joined string + delimiter)
123+
const selectList = (result as any).subquery.node.select_list[0];
124+
const stringSplitChildren = selectList.children[0].children;
125+
expect(stringSplitChildren.length).toBe(2);
126+
expect(stringSplitChildren[0].value.value).toContain('§‡¶'); // Contains delimiter
127+
});
128+
129+
it('Should use delimiter to join values', () => {
130+
const result = inTransform({
131+
member: 'country',
132+
operator: 'in',
133+
values: ['US', 'Canada'],
134+
memberInfo: {
135+
name: 'country',
136+
sql: 'table.country',
137+
type: 'string',
138+
},
139+
});
140+
141+
const selectList = (result as any).subquery.node.select_list[0];
142+
const joinedValue = selectList.children[0].children[0].value.value;
143+
const delimiter = selectList.children[0].children[1].value.value;
144+
145+
expect(delimiter).toBe('§‡¶');
146+
expect(joinedValue).toBe('US§‡¶Canada');
147+
});
148+
149+
it('Should handle the original test case structure for reference', () => {
150+
const output = inTransform({
151+
member: 'country',
152+
operator: 'in',
153+
values: ['US', 'Germany', 'Israel'],
154+
memberInfo: {
155+
name: 'country',
156+
sql: 'table.country',
157+
type: 'string_array',
158+
},
159+
}) as ConjunctionExpression;
160+
expect(output).toEqual({
161+
alias: '',
162+
catalog: '',
163+
children: [
164+
{
165+
alias: '',
166+
class: 'COLUMN_REF',
167+
column_names: ['country'],
168+
type: 'COLUMN_REF',
84169
},
170+
{
171+
alias: '',
172+
children: [
85173
{
86-
"alias": "",
87-
"children": [
88-
{
89-
"alias": "",
90-
"class": "CONSTANT",
91-
"type": "VALUE_CONSTANT",
92-
"value": {
93-
"is_null": false,
94-
"type": {
95-
"id": "VARCHAR",
96-
"type_info": null,
97-
},
98-
"value": "US",
174+
alias: '',
175+
class: 'CONSTANT',
176+
type: 'VALUE_CONSTANT',
177+
value: {
178+
is_null: false,
179+
type: {
180+
id: 'VARCHAR',
181+
type_info: null,
99182
},
183+
value: 'US',
184+
},
100185
},
101186
{
102-
"alias": "",
103-
"class": "CONSTANT",
104-
"type": "VALUE_CONSTANT",
105-
"value": {
106-
"is_null": false,
107-
"type": {
108-
"id": "VARCHAR",
109-
"type_info": null,
110-
},
111-
"value": "Germany",
187+
alias: '',
188+
class: 'CONSTANT',
189+
type: 'VALUE_CONSTANT',
190+
value: {
191+
is_null: false,
192+
type: {
193+
id: 'VARCHAR',
194+
type_info: null,
112195
},
196+
value: 'Germany',
197+
},
113198
},
114199
{
115-
"alias": "",
116-
"class": "CONSTANT",
117-
"type": "VALUE_CONSTANT",
118-
"value": {
119-
"is_null": false,
120-
"type": {
121-
"id": "VARCHAR",
122-
"type_info": null,
123-
},
124-
"value": "Israel",
200+
alias: '',
201+
class: 'CONSTANT',
202+
type: 'VALUE_CONSTANT',
203+
value: {
204+
is_null: false,
205+
type: {
206+
id: 'VARCHAR',
207+
type_info: null,
125208
},
126-
}],
127-
"class": "OPERATOR",
128-
"type": "ARRAY_CONSTRUCTOR",
129-
},
130-
],
131-
"class": "FUNCTION",
132-
"distinct": false,
133-
"export_state": false,
134-
"filter": null,
135-
"function_name": "&&",
136-
"is_operator": true,
137-
"order_bys": {
138-
"orders": [],
139-
"type": "ORDER_MODIFIER",
209+
value: 'Israel',
210+
},
211+
},
212+
],
213+
class: 'OPERATOR',
214+
type: 'ARRAY_CONSTRUCTOR',
140215
},
141-
"schema": "",
142-
"type": "FUNCTION",
216+
],
217+
class: 'FUNCTION',
218+
distinct: false,
219+
export_state: false,
220+
filter: null,
221+
function_name: '&&',
222+
is_operator: true,
223+
order_bys: {
224+
orders: [],
225+
type: 'ORDER_MODIFIER',
226+
},
227+
schema: '',
228+
type: 'FUNCTION',
143229
});
144230
});
231+
232+
it('Should throw error if values array is empty', () => {
233+
expect(() =>
234+
inTransform({
235+
member: 'country',
236+
operator: 'in',
237+
values: [],
238+
memberInfo: {
239+
name: 'country',
240+
sql: 'table.country',
241+
type: 'string',
242+
},
243+
})
244+
).toThrow('In filter must have at least one value');
245+
});
145246
});

0 commit comments

Comments
 (0)