Skip to content

Commit a492873

Browse files
pramodsatyaManoj Negi
pramodsatya
and
Manoj Negi
committed
[native] Add TestAggregations to native-tests
Co-authored-by: Manoj Negi <manojneg@in.ibm.com>
1 parent 96d6583 commit a492873

9 files changed

Lines changed: 335 additions & 12 deletions

File tree

‎presto-native-execution/src/test/java/com/facebook/presto/nativeworker/PrestoNativeQueryRunnerUtils.java‎

Lines changed: 10 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -130,7 +130,7 @@ public static QueryRunner createQueryRunner(
130130
defaultQueryRunner.close();
131131

132132
return createNativeQueryRunner(dataDirectory.get().toString(), prestoServerPath.get(), workerCount, cacheMaxSize, true, Optional.empty(),
133-
storageFormat, addStorageFormatToPath, false, isCoordinatorSidecarEnabled, false, enableRuntimeMetricsCollection, enableSsdCache, Collections.emptyMap());
133+
storageFormat, addStorageFormatToPath, false, isCoordinatorSidecarEnabled, false, enableRuntimeMetricsCollection, enableSsdCache, Collections.emptyMap(), Collections.emptyMap());
134134
}
135135

136136
public static QueryRunner createJavaQueryRunner()
@@ -338,7 +338,8 @@ public static QueryRunner createNativeQueryRunner(
338338
boolean singleNodeExecutionEnabled,
339339
boolean enableRuntimeMetricsCollection,
340340
boolean enableSsdCache,
341-
Map<String, String> extraProperties)
341+
Map<String, String> extraProperties,
342+
Map<String, String> extraCoordinatorProperties)
342343
throws Exception
343344
{
344345
// The property "hive.allow-drop-table" needs to be set to true because security is always "legacy" in NativeQueryRunner.
@@ -364,7 +365,7 @@ public static QueryRunner createNativeQueryRunner(
364365
.putAll(isCoordinatorSidecarEnabled ? getNativeSidecarProperties() : ImmutableMap.of())
365366
.putAll(extraProperties)
366367
.build(),
367-
coordinatorProperties.build(),
368+
coordinatorProperties.putAll(extraCoordinatorProperties).build(),
368369
"legacy",
369370
hiveProperties,
370371
workerCount,
@@ -424,7 +425,9 @@ public static QueryRunner createNativeQueryRunner(String remoteFunctionServerUds
424425
return createNativeQueryRunner(false, DEFAULT_STORAGE_FORMAT, Optional.ofNullable(remoteFunctionServerUds), false, false, false, false, false);
425426
}
426427

427-
public static QueryRunner createNativeQueryRunner(Map<String, String> extraProperties, String storageFormat)
428+
public static QueryRunner createNativeQueryRunner(String storageFormat,
429+
Map<String, String> extraProperties,
430+
Map<String, String> extraCoordinatorProperties)
428431
throws Exception
429432
{
430433
int cacheMaxSize = 0;
@@ -443,7 +446,8 @@ public static QueryRunner createNativeQueryRunner(Map<String, String> extraPrope
443446
false,
444447
false,
445448
false,
446-
extraProperties);
449+
extraProperties,
450+
extraCoordinatorProperties);
447451
}
448452

449453
public static QueryRunner createNativeQueryRunner(boolean useThrift)
@@ -491,6 +495,7 @@ public static QueryRunner createNativeQueryRunner(
491495
singleNodeExecutionEnabled,
492496
enableRuntimeMetricsCollection,
493497
enableSSDCache,
498+
Collections.emptyMap(),
494499
Collections.emptyMap());
495500
}
496501

‎presto-native-tests/src/test/java/com.facebook.presto.nativetests/AbstractTestAggregationsNative.java‎

Lines changed: 254 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,254 @@
1+
/*
2+
* Licensed under the Apache License, Version 2.0 (the "License");
3+
* you may not use this file except in compliance with the License.
4+
* You may obtain a copy of the License at
5+
*
6+
* http://www.apache.org/licenses/LICENSE-2.0
7+
*
8+
* Unless required by applicable law or agreed to in writing, software
9+
* distributed under the License is distributed on an "AS IS" BASIS,
10+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11+
* See the License for the specific language governing permissions and
12+
* limitations under the License.
13+
*/
14+
package com.facebook.presto.nativetests;
15+
16+
import com.facebook.presto.nativeworker.NativeQueryRunnerUtils;
17+
import com.facebook.presto.nativeworker.PrestoNativeQueryRunnerUtils;
18+
import com.facebook.presto.testing.QueryRunner;
19+
import com.facebook.presto.tests.AbstractTestAggregations;
20+
import org.testng.annotations.Parameters;
21+
import org.testng.annotations.Test;
22+
23+
import static java.lang.String.format;
24+
25+
public abstract class AbstractTestAggregationsNative
26+
extends AbstractTestAggregations
27+
{
28+
@Parameters("storageFormat")
29+
@Override
30+
protected void createTables()
31+
{
32+
try {
33+
String storageFormat = System.getProperty("storageFormat");
34+
QueryRunner javaQueryRunner = PrestoNativeQueryRunnerUtils.createJavaQueryRunner(storageFormat);
35+
if (storageFormat.equals("DWRF")) {
36+
NativeQueryRunnerUtils.createAllTables(javaQueryRunner, true);
37+
}
38+
else {
39+
NativeQueryRunnerUtils.createAllTables(javaQueryRunner, false);
40+
}
41+
javaQueryRunner.close();
42+
}
43+
catch (Exception e) {
44+
throw new RuntimeException(e);
45+
}
46+
}
47+
48+
@Parameters("storageFormat")
49+
@Override
50+
@Test
51+
public void testGroupByExtract()
52+
{
53+
String orderdate = System.getProperty("storageFormat").equals("DWRF") ? "cast(orderdate as DATE)" : "orderdate";
54+
55+
// whole expression in group by
56+
assertQuery(format("SELECT EXTRACT(YEAR FROM %s), count(*) FROM orders GROUP BY EXTRACT(YEAR FROM %s)", orderdate, orderdate));
57+
58+
assertQuery(
59+
format("SELECT EXTRACT(YEAR FROM %s), count(*) FROM orders GROUP BY 1", orderdate),
60+
format("SELECT EXTRACT(YEAR FROM %s), count(*) FROM orders GROUP BY EXTRACT(YEAR FROM %s)", orderdate, orderdate));
61+
62+
// argument in group by
63+
assertQuery(format("SELECT EXTRACT(YEAR FROM %s), count(*) FROM orders GROUP BY orderdate", orderdate));
64+
}
65+
66+
@Parameters("storageFormat")
67+
@Override
68+
@Test
69+
public void testGroupingSetMixedExpressionAndColumn()
70+
{
71+
String shipdate = System.getProperty("storageFormat").equals("DWRF") ? "cast(shipdate as DATE)" : "shipdate";
72+
73+
assertQuery(format("SELECT suppkey, month(%s), SUM(CAST(quantity AS BIGINT)) FROM lineitem GROUP BY month(%s), ROLLUP(suppkey)",
74+
shipdate, shipdate),
75+
format("SELECT suppkey, month(%s), SUM(CAST(quantity AS BIGINT)) FROM lineitem GROUP BY month(%s), suppkey UNION ALL " +
76+
"SELECT NULL, month(%s), SUM(CAST(quantity AS BIGINT)) FROM lineitem GROUP BY month(%s)",
77+
shipdate, shipdate, shipdate, shipdate));
78+
}
79+
80+
@Parameters("storageFormat")
81+
@Override
82+
@Test
83+
public void testGroupingSetMixedExpressionAndOrdinal()
84+
{
85+
String shipdate = System.getProperty("storageFormat").equals("DWRF") ? "cast(shipdate as DATE)" : "shipdate";
86+
87+
assertQuery(format("SELECT suppkey, month(%s), SUM(CAST(quantity AS BIGINT)) FROM lineitem GROUP BY 2, ROLLUP(suppkey)", shipdate),
88+
format("SELECT suppkey, month(%s), SUM(CAST(quantity AS BIGINT)) FROM lineitem GROUP BY month(%s), suppkey UNION ALL " +
89+
"SELECT NULL, month(%s), SUM(CAST(quantity AS BIGINT)) FROM lineitem GROUP BY month(%s)",
90+
shipdate, shipdate, shipdate, shipdate));
91+
}
92+
93+
/// `approx_distinct` aggregate function returns a different value for certain datatypes in Presto C++, and the
94+
/// function does not support arguments of type `TIMESTAMP WITH TIME ZONE` (see issue for more details:
95+
/// https://github.com/prestodb/presto/issues/24815). Presto C++ does not support datatypes `CHAR` and `TIME`, see:
96+
/// https://github.com/prestodb/presto/blob/master/presto-docs/src/main/sphinx/presto_cpp/limitations.rst.
97+
@Parameters("storageFormat")
98+
@Override
99+
@Test
100+
public void testApproximateCountDistinct()
101+
{
102+
String timeTypeUnsupportedError = "Failed to parse type.*time";
103+
String charTypeUnsupportedError = "Failed to parse type.*char";
104+
String signatureUnsupportedError = ".*Aggregate function signature is not supported.*";
105+
106+
// test NULL
107+
assertQuery("SELECT approx_distinct(NULL)", "SELECT 0");
108+
assertQuery("SELECT approx_distinct(NULL, 0.023)", "SELECT 0");
109+
110+
// test date
111+
String orderdate = System.getProperty("storageFormat").equals("DWRF") ? "cast(orderdate as DATE)" : "orderdate";
112+
assertQuery(format("SELECT approx_distinct(%s) FROM orders", orderdate), "SELECT 2372");
113+
assertQuery(format("SELECT approx_distinct(%s, 0.023) FROM orders", orderdate), "SELECT 2372");
114+
115+
// test timestamp
116+
assertQuery("SELECT approx_distinct(CAST(orderdate AS TIMESTAMP)) FROM orders", "SELECT 2347");
117+
assertQuery("SELECT approx_distinct(CAST(orderdate AS TIMESTAMP), 0.023) FROM orders", "SELECT 2347");
118+
119+
// test timestamp with time zone
120+
assertQueryFails("SELECT approx_distinct(CAST(orderdate AS TIMESTAMP WITH TIME ZONE)) FROM orders",
121+
signatureUnsupportedError, true);
122+
assertQueryFails("SELECT approx_distinct(CAST(orderdate AS TIMESTAMP WITH TIME ZONE), 0.023) FROM orders",
123+
signatureUnsupportedError, true);
124+
125+
// test time
126+
assertQueryFails("SELECT approx_distinct(CAST(from_unixtime(custkey) AS TIME)) FROM orders", timeTypeUnsupportedError, true);
127+
assertQueryFails("SELECT approx_distinct(CAST(from_unixtime(custkey) AS TIME), 0.023) FROM orders", timeTypeUnsupportedError, true);
128+
129+
// test time with time zone
130+
assertQueryFails("SELECT approx_distinct(CAST(from_unixtime(custkey) AS TIME WITH TIME ZONE)) FROM orders", timeTypeUnsupportedError, true);
131+
assertQueryFails("SELECT approx_distinct(CAST(from_unixtime(custkey) AS TIME WITH TIME ZONE), 0.023) FROM orders", timeTypeUnsupportedError, true);
132+
133+
// test short decimal
134+
assertQuery("SELECT approx_distinct(CAST(custkey AS DECIMAL(18, 0))) FROM orders", "SELECT 990");
135+
assertQuery("SELECT approx_distinct(CAST(custkey AS DECIMAL(18, 0)), 0.023) FROM orders", "SELECT 990");
136+
137+
// test long decimal
138+
assertQuery("SELECT approx_distinct(CAST(custkey AS DECIMAL(25, 20))) FROM orders", "SELECT 1013");
139+
assertQuery("SELECT approx_distinct(CAST(custkey AS DECIMAL(25, 20)), 0.023) FROM orders", "SELECT 1013");
140+
141+
// test real
142+
assertQuery("SELECT approx_distinct(CAST(custkey AS REAL)) FROM orders", "SELECT 982");
143+
assertQuery("SELECT approx_distinct(CAST(custkey AS REAL), 0.023) FROM orders", "SELECT 982");
144+
145+
// test bigint
146+
assertQuery("SELECT approx_distinct(custkey) FROM orders", "SELECT 990");
147+
assertQuery("SELECT approx_distinct(custkey, 0.023) FROM orders", "SELECT 990");
148+
149+
// test integer
150+
assertQuery("SELECT approx_distinct(CAST(custkey AS INTEGER)) FROM orders", "SELECT 1028");
151+
assertQuery("SELECT approx_distinct(CAST(custkey AS INTEGER), 0.023) FROM orders", "SELECT 1028");
152+
153+
// test smallint
154+
assertQuery("SELECT approx_distinct(CAST(custkey AS SMALLINT)) FROM orders", "SELECT 1023");
155+
assertQuery("SELECT approx_distinct(CAST(custkey AS SMALLINT), 0.023) FROM orders", "SELECT 1023");
156+
157+
// test tinyint
158+
assertQuery("SELECT approx_distinct(CAST((custkey % 128) AS TINYINT)) FROM orders", "SELECT 128");
159+
assertQuery("SELECT approx_distinct(CAST((custkey % 128) AS TINYINT), 0.023) FROM orders", "SELECT 128");
160+
161+
// test double
162+
assertQuery("SELECT approx_distinct(CAST(custkey AS DOUBLE)) FROM orders", "SELECT 1014");
163+
assertQuery("SELECT approx_distinct(CAST(custkey AS DOUBLE), 0.023) FROM orders", "SELECT 1014");
164+
165+
// test varchar
166+
assertQuery("SELECT approx_distinct(CAST(custkey AS VARCHAR)) FROM orders", "SELECT 1036");
167+
assertQuery("SELECT approx_distinct(CAST(custkey AS VARCHAR), 0.023) FROM orders", "SELECT 1036");
168+
169+
// test char
170+
assertQueryFails("SELECT approx_distinct(CAST(CAST(custkey AS VARCHAR) AS CHAR(20))) FROM orders", charTypeUnsupportedError, true);
171+
assertQueryFails("SELECT approx_distinct(CAST(CAST(custkey AS VARCHAR) AS CHAR(20)), 0.023) FROM orders", charTypeUnsupportedError, true);
172+
173+
// test varbinary
174+
assertQuery("SELECT approx_distinct(to_utf8(CAST(custkey AS VARCHAR))) FROM orders", "SELECT 1036");
175+
assertQuery("SELECT approx_distinct(to_utf8(CAST(custkey AS VARCHAR)), 0.023) FROM orders", "SELECT 1036");
176+
}
177+
178+
/// `sum_data_size_for_stats` returns a different value for `Varchar` and `Varbinary` datatypes in Presto C++, see:
179+
/// https://github.com/prestodb/presto/issues/20909. `CHAR` datatype is not supported in Presto C++, see issue:
180+
/// https://github.com/prestodb/presto/issues/21332.
181+
@Override
182+
@Test
183+
public void testSumDataSizeForStats()
184+
{
185+
// varchar
186+
assertQuery("SELECT \"sum_data_size_for_stats\"(comment) FROM orders", "SELECT 787364");
187+
188+
// char
189+
// Presto removes trailing whitespaces when casting to CHAR.
190+
// Hard code the expected data size since there is no easy to way to compute it in H2.
191+
assertQueryFails("SELECT \"sum_data_size_for_stats\"(CAST(comment AS CHAR(1000))) FROM orders",
192+
"Failed to parse type \\[char\\(1000\\)]", true);
193+
194+
// varbinary
195+
assertQuery("SELECT \"sum_data_size_for_stats\"(CAST(comment AS VARBINARY)) FROM orders", "SELECT 787364");
196+
197+
// array
198+
assertQuery("SELECT \"sum_data_size_for_stats\"(ARRAY[comment]) FROM orders", "SELECT 847364");
199+
assertQuery("SELECT \"sum_data_size_for_stats\"(ARRAY[comment, comment]) FROM orders", "SELECT 1634728");
200+
201+
// map
202+
assertQuery("SELECT \"sum_data_size_for_stats\"(map(ARRAY[1], ARRAY[comment])) FROM orders", "SELECT 907364");
203+
assertQuery("SELECT \"sum_data_size_for_stats\"(map(ARRAY[1, 2], ARRAY[comment, comment])) FROM orders", "SELECT 1754728");
204+
205+
// row
206+
assertQuery("SELECT \"sum_data_size_for_stats\"(ROW(comment)) FROM orders", "SELECT 847364");
207+
assertQuery("SELECT \"sum_data_size_for_stats\"(ROW(comment, comment)) FROM orders", "SELECT 1634728");
208+
}
209+
210+
/// `max_data_size_for_stats` returns a different value for `Varchar` and `Varbinary` datatypes in Presto C++, see:
211+
/// https://github.com/prestodb/presto/issues/20909. `CHAR` datatype is not supported in Presto C++, see issue:
212+
/// https://github.com/prestodb/presto/issues/21332.
213+
@Override
214+
@Test
215+
public void testMaxDataSizeForStats()
216+
{
217+
// varchar
218+
assertQuery("SELECT \"max_data_size_for_stats\"(comment) FROM orders", "select 82");
219+
220+
// char
221+
assertQueryFails("SELECT \"max_data_size_for_stats\"(CAST(comment AS CHAR(1000))) FROM orders",
222+
"Failed to parse type \\[char\\(1000\\)]", true);
223+
224+
// varbinary
225+
assertQuery("SELECT \"max_data_size_for_stats\"(CAST(comment AS VARBINARY)) FROM orders", "select 82");
226+
227+
// max_data_size_for_stats is not needed for array, map and row
228+
}
229+
230+
/// Function `tdigest_agg` is not supported in Presto C++, see: https://github.com/prestodb/presto/issues/24811.
231+
/// `qdigest` datatype is not supported in Presto C++, see: https://github.com/prestodb/presto/issues/24814.
232+
@Override
233+
@Test(enabled = false, dataProvider = "getType")
234+
public void testStatisticalDigest(String type) {}
235+
236+
/// Function `tdigest_agg` is not supported in Presto C++, see: https://github.com/prestodb/presto/issues/24811.
237+
/// `qdigest` datatype is not supported in Presto C++, see: https://github.com/prestodb/presto/issues/24814.
238+
@Override
239+
@Test(enabled = false, dataProvider = "getType")
240+
public void testStatisticalDigestGroupBy(String type) {}
241+
242+
/// Function `tdigest_agg` is not supported in Presto C++, see: https://github.com/prestodb/presto/issues/24811.
243+
/// `qdigest` datatype is not supported in Presto C++, see: https://github.com/prestodb/presto/issues/24814.
244+
@Override
245+
@Test(enabled = false, dataProvider = "getType")
246+
public void testStatisticalDigestMerge(String type) {}
247+
248+
/// Aggregate function `merge` is not supported for `tdigest` type in Presto C++, see issue for more details:
249+
/// https://github.com/prestodb/presto/issues/24813. `qdigest` datatype is not supported in Presto C++, see:
250+
/// https://github.com/prestodb/presto/issues/24814.
251+
@Override
252+
@Test(enabled = false, dataProvider = "getType")
253+
public void testStatisticalDigestMergeGroupBy(String type) {}
254+
}

‎presto-native-tests/src/test/java/com.facebook.presto.nativetests/TestAggregations.java‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
/*
2+
* Licensed under the Apache License, Version 2.0 (the "License");
3+
* you may not use this file except in compliance with the License.
4+
* You may obtain a copy of the License at
5+
*
6+
* http://www.apache.org/licenses/LICENSE-2.0
7+
*
8+
* Unless required by applicable law or agreed to in writing, software
9+
* distributed under the License is distributed on an "AS IS" BASIS,
10+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11+
* See the License for the specific language governing permissions and
12+
* limitations under the License.
13+
*/
14+
package com.facebook.presto.nativetests;
15+
16+
import com.facebook.presto.nativeworker.PrestoNativeQueryRunnerUtils;
17+
import com.facebook.presto.testing.QueryRunner;
18+
import com.google.common.collect.ImmutableMap;
19+
import org.testng.annotations.Parameters;
20+
21+
public class TestAggregations
22+
extends AbstractTestAggregationsNative
23+
{
24+
@Parameters("storageFormat")
25+
@Override
26+
protected QueryRunner createQueryRunner() throws Exception
27+
{
28+
return PrestoNativeQueryRunnerUtils.createNativeQueryRunner(System.getProperty("storageFormat"), ImmutableMap.of(), ImmutableMap.of());
29+
}
30+
}

‎presto-native-tests/src/test/java/com.facebook.presto.nativetests/TestDistributedEngineOnlyQueries.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@ public class TestDistributedEngineOnlyQueries
3939
@Override
4040
protected QueryRunner createQueryRunner() throws Exception
4141
{
42-
return PrestoNativeQueryRunnerUtils.createNativeQueryRunner(ImmutableMap.of(), System.getProperty("storageFormat"));
42+
return PrestoNativeQueryRunnerUtils.createNativeQueryRunner(System.getProperty("storageFormat"), ImmutableMap.of(), ImmutableMap.of());
4343
}
4444

4545
@Parameters("storageFormat")

‎presto-native-tests/src/test/java/com.facebook.presto.nativetests/TestOptimizeMixedDistinctAggregations.java‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
2+
/*
3+
* Licensed under the Apache License, Version 2.0 (the "License");
4+
* you may not use this file except in compliance with the License.
5+
* You may obtain a copy of the License at
6+
*
7+
* http://www.apache.org/licenses/LICENSE-2.0
8+
*
9+
* Unless required by applicable law or agreed to in writing, software
10+
* distributed under the License is distributed on an "AS IS" BASIS,
11+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
* See the License for the specific language governing permissions and
13+
* limitations under the License.
14+
*/
15+
package com.facebook.presto.nativetests;
16+
17+
import com.facebook.presto.nativeworker.PrestoNativeQueryRunnerUtils;
18+
import com.facebook.presto.testing.QueryRunner;
19+
import com.google.common.collect.ImmutableMap;
20+
import org.testng.annotations.Parameters;
21+
22+
public class TestOptimizeMixedDistinctAggregations
23+
extends AbstractTestAggregationsNative
24+
{
25+
@Parameters("storageFormat")
26+
@Override
27+
protected QueryRunner createQueryRunner()
28+
throws Exception
29+
{
30+
return PrestoNativeQueryRunnerUtils.createNativeQueryRunner(
31+
System.getProperty("storageFormat"),
32+
ImmutableMap.of(),
33+
ImmutableMap.of("optimizer.optimize-mixed-distinct-aggregations", "true"));
34+
}
35+
}

‎presto-native-tests/src/test/java/com.facebook.presto.nativetests/TestOrderByQueries.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ public class TestOrderByQueries
2828
@Override
2929
protected QueryRunner createQueryRunner() throws Exception
3030
{
31-
return PrestoNativeQueryRunnerUtils.createNativeQueryRunner(ImmutableMap.of(), System.getProperty("storageFormat"));
31+
return PrestoNativeQueryRunnerUtils.createNativeQueryRunner(System.getProperty("storageFormat"), ImmutableMap.of(), ImmutableMap.of());
3232
}
3333

3434
@Parameters("storageFormat")

0 commit comments

Comments
 (0)