Skip to content

Commit 0a56b46

Browse files
pramodsatyaManoj Negi
pramodsatya
and
Manoj Negi
committed
[native] Add TestAggregations to native-tests
Co-authored-by: Manoj Negi <manojneg@in.ibm.com>
1 parent 1e45c1b commit 0a56b46

9 files changed

Lines changed: 290 additions & 11 deletions

File tree

‎presto-native-execution/src/test/java/com/facebook/presto/nativeworker/PrestoNativeQueryRunnerUtils.java‎

Lines changed: 10 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -130,7 +130,7 @@ public static QueryRunner createQueryRunner(
130130
defaultQueryRunner.close();
131131

132132
return createNativeQueryRunner(dataDirectory.get().toString(), prestoServerPath.get(), workerCount, cacheMaxSize, true, Optional.empty(),
133-
storageFormat, addStorageFormatToPath, false, isCoordinatorSidecarEnabled, false, enableRuntimeMetricsCollection, enableSsdCache, Collections.emptyMap());
133+
storageFormat, addStorageFormatToPath, false, isCoordinatorSidecarEnabled, false, enableRuntimeMetricsCollection, enableSsdCache, Collections.emptyMap(), Collections.emptyMap());
134134
}
135135

136136
public static QueryRunner createJavaQueryRunner()
@@ -338,7 +338,8 @@ public static QueryRunner createNativeQueryRunner(
338338
boolean singleNodeExecutionEnabled,
339339
boolean enableRuntimeMetricsCollection,
340340
boolean enableSsdCache,
341-
Map<String, String> extraProperties)
341+
Map<String, String> extraProperties,
342+
Map<String, String> extraCoordinatorProperties)
342343
throws Exception
343344
{
344345
// The property "hive.allow-drop-table" needs to be set to true because security is always "legacy" in NativeQueryRunner.
@@ -364,7 +365,7 @@ public static QueryRunner createNativeQueryRunner(
364365
.putAll(isCoordinatorSidecarEnabled ? getNativeSidecarProperties() : ImmutableMap.of())
365366
.putAll(extraProperties)
366367
.build(),
367-
coordinatorProperties.build(),
368+
coordinatorProperties.putAll(extraCoordinatorProperties).build(),
368369
"legacy",
369370
hiveProperties,
370371
workerCount,
@@ -424,7 +425,9 @@ public static QueryRunner createNativeQueryRunner(String remoteFunctionServerUds
424425
return createNativeQueryRunner(false, DEFAULT_STORAGE_FORMAT, Optional.ofNullable(remoteFunctionServerUds), false, false, false, false, false);
425426
}
426427

427-
public static QueryRunner createNativeQueryRunner(Map<String, String> extraProperties, String storageFormat)
428+
public static QueryRunner createNativeQueryRunner(String storageFormat,
429+
Map<String, String> extraProperties,
430+
Map<String, String> extraCoordinatorProperties)
428431
throws Exception
429432
{
430433
int cacheMaxSize = 0;
@@ -443,7 +446,8 @@ public static QueryRunner createNativeQueryRunner(Map<String, String> extraPrope
443446
false,
444447
false,
445448
false,
446-
extraProperties);
449+
extraProperties,
450+
extraCoordinatorProperties);
447451
}
448452

449453
public static QueryRunner createNativeQueryRunner(boolean useThrift)
@@ -491,6 +495,7 @@ public static QueryRunner createNativeQueryRunner(
491495
singleNodeExecutionEnabled,
492496
enableRuntimeMetricsCollection,
493497
enableSSDCache,
498+
Collections.emptyMap(),
494499
Collections.emptyMap());
495500
}
496501

‎presto-native-tests/src/test/java/com.facebook.presto.nativetests/AbstractTestAggregationsNative.java‎

Lines changed: 209 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,209 @@
1+
/*
2+
* Licensed under the Apache License, Version 2.0 (the "License");
3+
* you may not use this file except in compliance with the License.
4+
* You may obtain a copy of the License at
5+
*
6+
* http://www.apache.org/licenses/LICENSE-2.0
7+
*
8+
* Unless required by applicable law or agreed to in writing, software
9+
* distributed under the License is distributed on an "AS IS" BASIS,
10+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11+
* See the License for the specific language governing permissions and
12+
* limitations under the License.
13+
*/
14+
package com.facebook.presto.nativetests;
15+
16+
import com.facebook.presto.nativeworker.NativeQueryRunnerUtils;
17+
import com.facebook.presto.nativeworker.PrestoNativeQueryRunnerUtils;
18+
import com.facebook.presto.testing.QueryRunner;
19+
import com.facebook.presto.tests.AbstractTestAggregations;
20+
import org.testng.annotations.Parameters;
21+
import org.testng.annotations.Test;
22+
23+
import static java.lang.String.format;
24+
25+
public abstract class AbstractTestAggregationsNative
26+
extends AbstractTestAggregations
27+
{
28+
@Parameters("storageFormat")
29+
@Override
30+
protected void createTables()
31+
{
32+
try {
33+
String storageFormat = System.getProperty("storageFormat");
34+
QueryRunner javaQueryRunner = PrestoNativeQueryRunnerUtils.createJavaQueryRunner(storageFormat);
35+
if (storageFormat.equals("DWRF")) {
36+
NativeQueryRunnerUtils.createAllTables(javaQueryRunner, true);
37+
}
38+
else {
39+
NativeQueryRunnerUtils.createAllTables(javaQueryRunner, false);
40+
}
41+
javaQueryRunner.close();
42+
}
43+
catch (Exception e) {
44+
throw new RuntimeException(e);
45+
}
46+
}
47+
48+
/// `approx_distinct` aggregate function returns a different value for certain datatypes in Presto C++, and the
49+
/// function does not support arguments of type `TIMESTAMP WITH TIME ZONE` (see issue for more details:
50+
/// https://github.com/prestodb/presto/issues/24815). Presto C++ does not support datatypes `CHAR` and `TIME`, see:
51+
/// https://github.com/prestodb/presto/blob/master/presto-docs/src/main/sphinx/presto_cpp/limitations.rst.
52+
@Parameters("storageFormat")
53+
@Override
54+
@Test
55+
public void testApproximateCountDistinct()
56+
{
57+
String timeTypeUnsupportedError = "Failed to parse type.*time";
58+
String charTypeUnsupportedError = "Failed to parse type.*char";
59+
String signatureUnsupportedError = ".*Aggregate function signature is not supported.*";
60+
61+
// test NULL
62+
assertQuery("SELECT approx_distinct(NULL)", "SELECT 0");
63+
assertQuery("SELECT approx_distinct(NULL, 0.023)", "SELECT 0");
64+
65+
// test date
66+
String orderdate = System.getProperty("storageFormat").equals("DWRF") ? "cast(orderdate as DATE)" : "orderdate";
67+
assertQuery(format("SELECT approx_distinct(%s) FROM orders", orderdate), "SELECT 2372");
68+
assertQuery(format("SELECT approx_distinct(%s, 0.023) FROM orders", orderdate), "SELECT 2372");
69+
70+
// test timestamp
71+
assertQuery("SELECT approx_distinct(CAST(orderdate AS TIMESTAMP)) FROM orders", "SELECT 2347");
72+
assertQuery("SELECT approx_distinct(CAST(orderdate AS TIMESTAMP), 0.023) FROM orders", "SELECT 2347");
73+
74+
// test timestamp with time zone
75+
assertQueryFails("SELECT approx_distinct(CAST(orderdate AS TIMESTAMP WITH TIME ZONE)) FROM orders",
76+
signatureUnsupportedError, true);
77+
assertQueryFails("SELECT approx_distinct(CAST(orderdate AS TIMESTAMP WITH TIME ZONE), 0.023) FROM orders",
78+
signatureUnsupportedError, true);
79+
80+
// test time
81+
assertQueryFails("SELECT approx_distinct(CAST(from_unixtime(custkey) AS TIME)) FROM orders", timeTypeUnsupportedError, true);
82+
assertQueryFails("SELECT approx_distinct(CAST(from_unixtime(custkey) AS TIME), 0.023) FROM orders", timeTypeUnsupportedError, true);
83+
84+
// test time with time zone
85+
assertQueryFails("SELECT approx_distinct(CAST(from_unixtime(custkey) AS TIME WITH TIME ZONE)) FROM orders", timeTypeUnsupportedError, true);
86+
assertQueryFails("SELECT approx_distinct(CAST(from_unixtime(custkey) AS TIME WITH TIME ZONE), 0.023) FROM orders", timeTypeUnsupportedError, true);
87+
88+
// test short decimal
89+
assertQuery("SELECT approx_distinct(CAST(custkey AS DECIMAL(18, 0))) FROM orders", "SELECT 990");
90+
assertQuery("SELECT approx_distinct(CAST(custkey AS DECIMAL(18, 0)), 0.023) FROM orders", "SELECT 990");
91+
92+
// test long decimal
93+
assertQuery("SELECT approx_distinct(CAST(custkey AS DECIMAL(25, 20))) FROM orders", "SELECT 1013");
94+
assertQuery("SELECT approx_distinct(CAST(custkey AS DECIMAL(25, 20)), 0.023) FROM orders", "SELECT 1013");
95+
96+
// test real
97+
assertQuery("SELECT approx_distinct(CAST(custkey AS REAL)) FROM orders", "SELECT 982");
98+
assertQuery("SELECT approx_distinct(CAST(custkey AS REAL), 0.023) FROM orders", "SELECT 982");
99+
100+
// test bigint
101+
assertQuery("SELECT approx_distinct(custkey) FROM orders", "SELECT 990");
102+
assertQuery("SELECT approx_distinct(custkey, 0.023) FROM orders", "SELECT 990");
103+
104+
// test integer
105+
assertQuery("SELECT approx_distinct(CAST(custkey AS INTEGER)) FROM orders", "SELECT 1028");
106+
assertQuery("SELECT approx_distinct(CAST(custkey AS INTEGER), 0.023) FROM orders", "SELECT 1028");
107+
108+
// test smallint
109+
assertQuery("SELECT approx_distinct(CAST(custkey AS SMALLINT)) FROM orders", "SELECT 1023");
110+
assertQuery("SELECT approx_distinct(CAST(custkey AS SMALLINT), 0.023) FROM orders", "SELECT 1023");
111+
112+
// test tinyint
113+
assertQuery("SELECT approx_distinct(CAST((custkey % 128) AS TINYINT)) FROM orders", "SELECT 128");
114+
assertQuery("SELECT approx_distinct(CAST((custkey % 128) AS TINYINT), 0.023) FROM orders", "SELECT 128");
115+
116+
// test double
117+
assertQuery("SELECT approx_distinct(CAST(custkey AS DOUBLE)) FROM orders", "SELECT 1014");
118+
assertQuery("SELECT approx_distinct(CAST(custkey AS DOUBLE), 0.023) FROM orders", "SELECT 1014");
119+
120+
// test varchar
121+
assertQuery("SELECT approx_distinct(CAST(custkey AS VARCHAR)) FROM orders", "SELECT 1036");
122+
assertQuery("SELECT approx_distinct(CAST(custkey AS VARCHAR), 0.023) FROM orders", "SELECT 1036");
123+
124+
// test char
125+
assertQueryFails("SELECT approx_distinct(CAST(CAST(custkey AS VARCHAR) AS CHAR(20))) FROM orders", charTypeUnsupportedError, true);
126+
assertQueryFails("SELECT approx_distinct(CAST(CAST(custkey AS VARCHAR) AS CHAR(20)), 0.023) FROM orders", charTypeUnsupportedError, true);
127+
128+
// test varbinary
129+
assertQuery("SELECT approx_distinct(to_utf8(CAST(custkey AS VARCHAR))) FROM orders", "SELECT 1036");
130+
assertQuery("SELECT approx_distinct(to_utf8(CAST(custkey AS VARCHAR)), 0.023) FROM orders", "SELECT 1036");
131+
}
132+
133+
/// `sum_data_size_for_stats` returns a different value for `Varchar` and `Varbinary` datatypes in Presto C++, see:
134+
/// https://github.com/prestodb/presto/issues/20909. `CHAR` datatype is not supported in Presto C++, see issue:
135+
/// https://github.com/prestodb/presto/issues/21332.
136+
@Override
137+
@Test
138+
public void testSumDataSizeForStats()
139+
{
140+
// varchar
141+
assertQuery("SELECT \"sum_data_size_for_stats\"(comment) FROM orders", "SELECT 787364");
142+
143+
// char
144+
// Presto removes trailing whitespaces when casting to CHAR.
145+
// Hard code the expected data size since there is no easy to way to compute it in H2.
146+
assertQueryFails("SELECT \"sum_data_size_for_stats\"(CAST(comment AS CHAR(1000))) FROM orders",
147+
"Failed to parse type \\[char\\(1000\\)]", true);
148+
149+
// varbinary
150+
assertQuery("SELECT \"sum_data_size_for_stats\"(CAST(comment AS VARBINARY)) FROM orders", "SELECT 787364");
151+
152+
// array
153+
assertQuery("SELECT \"sum_data_size_for_stats\"(ARRAY[comment]) FROM orders", "SELECT 847364");
154+
assertQuery("SELECT \"sum_data_size_for_stats\"(ARRAY[comment, comment]) FROM orders", "SELECT 1634728");
155+
156+
// map
157+
assertQuery("SELECT \"sum_data_size_for_stats\"(map(ARRAY[1], ARRAY[comment])) FROM orders", "SELECT 907364");
158+
assertQuery("SELECT \"sum_data_size_for_stats\"(map(ARRAY[1, 2], ARRAY[comment, comment])) FROM orders", "SELECT 1754728");
159+
160+
// row
161+
assertQuery("SELECT \"sum_data_size_for_stats\"(ROW(comment)) FROM orders", "SELECT 847364");
162+
assertQuery("SELECT \"sum_data_size_for_stats\"(ROW(comment, comment)) FROM orders", "SELECT 1634728");
163+
}
164+
165+
/// `max_data_size_for_stats` returns a different value for `Varchar` and `Varbinary` datatypes in Presto C++, see:
166+
/// https://github.com/prestodb/presto/issues/20909. `CHAR` datatype is not supported in Presto C++, see issue:
167+
/// https://github.com/prestodb/presto/issues/21332.
168+
@Override
169+
@Test
170+
public void testMaxDataSizeForStats()
171+
{
172+
// varchar
173+
assertQuery("SELECT \"max_data_size_for_stats\"(comment) FROM orders", "select 82");
174+
175+
// char
176+
assertQueryFails("SELECT \"max_data_size_for_stats\"(CAST(comment AS CHAR(1000))) FROM orders",
177+
"Failed to parse type \\[char\\(1000\\)]", true);
178+
179+
// varbinary
180+
assertQuery("SELECT \"max_data_size_for_stats\"(CAST(comment AS VARBINARY)) FROM orders", "select 82");
181+
182+
// max_data_size_for_stats is not needed for array, map and row
183+
}
184+
185+
/// Function `tdigest_agg` is not supported in Presto C++, see: https://github.com/prestodb/presto/issues/24811.
186+
/// `qdigest` datatype is not supported in Presto C++, see: https://github.com/prestodb/presto/issues/24814.
187+
@Override
188+
@Test(enabled = false, dataProvider = "getType")
189+
public void testStatisticalDigest(String type) {}
190+
191+
/// Function `tdigest_agg` is not supported in Presto C++, see: https://github.com/prestodb/presto/issues/24811.
192+
/// `qdigest` datatype is not supported in Presto C++, see: https://github.com/prestodb/presto/issues/24814.
193+
@Override
194+
@Test(enabled = false, dataProvider = "getType")
195+
public void testStatisticalDigestGroupBy(String type) {}
196+
197+
/// Function `tdigest_agg` is not supported in Presto C++, see: https://github.com/prestodb/presto/issues/24811.
198+
/// `qdigest` datatype is not supported in Presto C++, see: https://github.com/prestodb/presto/issues/24814.
199+
@Override
200+
@Test(enabled = false, dataProvider = "getType")
201+
public void testStatisticalDigestMerge(String type) {}
202+
203+
/// Aggregate function `merge` is not supported for `tdigest` type in Presto C++, see issue for more details:
204+
/// https://github.com/prestodb/presto/issues/24813. `qdigest` datatype is not supported in Presto C++, see:
205+
/// https://github.com/prestodb/presto/issues/24814.
206+
@Override
207+
@Test(enabled = false, dataProvider = "getType")
208+
public void testStatisticalDigestMergeGroupBy(String type) {}
209+
}

‎presto-native-tests/src/test/java/com.facebook.presto.nativetests/TestAggregations.java‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
/*
2+
* Licensed under the Apache License, Version 2.0 (the "License");
3+
* you may not use this file except in compliance with the License.
4+
* You may obtain a copy of the License at
5+
*
6+
* http://www.apache.org/licenses/LICENSE-2.0
7+
*
8+
* Unless required by applicable law or agreed to in writing, software
9+
* distributed under the License is distributed on an "AS IS" BASIS,
10+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11+
* See the License for the specific language governing permissions and
12+
* limitations under the License.
13+
*/
14+
package com.facebook.presto.nativetests;
15+
16+
import com.facebook.presto.nativeworker.PrestoNativeQueryRunnerUtils;
17+
import com.facebook.presto.testing.QueryRunner;
18+
import com.google.common.collect.ImmutableMap;
19+
20+
public class TestAggregations
21+
extends AbstractTestAggregationsNative
22+
{
23+
@Override
24+
protected QueryRunner createQueryRunner() throws Exception
25+
{
26+
return PrestoNativeQueryRunnerUtils.createNativeQueryRunner(System.getProperty("storageFormat"),
27+
ImmutableMap.of(), ImmutableMap.of());
28+
}
29+
}

‎presto-native-tests/src/test/java/com.facebook.presto.nativetests/TestDistributedEngineOnlyQueries.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,8 @@ public class TestDistributedEngineOnlyQueries
3939
@Override
4040
protected QueryRunner createQueryRunner() throws Exception
4141
{
42-
return PrestoNativeQueryRunnerUtils.createNativeQueryRunner(ImmutableMap.of(), System.getProperty("storageFormat"));
42+
return PrestoNativeQueryRunnerUtils.createNativeQueryRunner(System.getProperty("storageFormat"),
43+
ImmutableMap.of(), ImmutableMap.of());
4344
}
4445

4546
@Parameters("storageFormat")

‎presto-native-tests/src/test/java/com.facebook.presto.nativetests/TestOptimizeMixedDistinctAggregations.java‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
2+
/*
3+
* Licensed under the Apache License, Version 2.0 (the "License");
4+
* you may not use this file except in compliance with the License.
5+
* You may obtain a copy of the License at
6+
*
7+
* http://www.apache.org/licenses/LICENSE-2.0
8+
*
9+
* Unless required by applicable law or agreed to in writing, software
10+
* distributed under the License is distributed on an "AS IS" BASIS,
11+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
* See the License for the specific language governing permissions and
13+
* limitations under the License.
14+
*/
15+
package com.facebook.presto.nativetests;
16+
17+
import com.facebook.presto.nativeworker.PrestoNativeQueryRunnerUtils;
18+
import com.facebook.presto.testing.QueryRunner;
19+
import com.google.common.collect.ImmutableMap;
20+
21+
public class TestOptimizeMixedDistinctAggregations
22+
extends AbstractTestAggregationsNative
23+
{
24+
@Override
25+
protected QueryRunner createQueryRunner()
26+
throws Exception
27+
{
28+
return PrestoNativeQueryRunnerUtils.createNativeQueryRunner(System.getProperty("storageFormat"),
29+
ImmutableMap.of(),
30+
ImmutableMap.of("optimizer.optimize-mixed-distinct-aggregations", "true"));
31+
}
32+
}

‎presto-native-tests/src/test/java/com.facebook.presto.nativetests/TestOrderByQueries.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,8 @@ public class TestOrderByQueries
2828
@Override
2929
protected QueryRunner createQueryRunner() throws Exception
3030
{
31-
return PrestoNativeQueryRunnerUtils.createNativeQueryRunner(ImmutableMap.of(), System.getProperty("storageFormat"));
31+
return PrestoNativeQueryRunnerUtils.createNativeQueryRunner(System.getProperty("storageFormat"),
32+
ImmutableMap.of(), ImmutableMap.of());
3233
}
3334

3435
@Parameters("storageFormat")

‎presto-native-tests/src/test/java/com.facebook.presto.nativetests/TestRepartitionQueries.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,8 @@ public class TestRepartitionQueries
2727
@Override
2828
protected QueryRunner createQueryRunner() throws Exception
2929
{
30-
return PrestoNativeQueryRunnerUtils.createNativeQueryRunner(ImmutableMap.of(), System.getProperty("storageFormat"));
30+
return PrestoNativeQueryRunnerUtils.createNativeQueryRunner(System.getProperty("storageFormat"),
31+
ImmutableMap.of(), ImmutableMap.of());
3132
}
3233

3334
@Parameters("storageFormat")

‎presto-native-tests/src/test/java/com.facebook.presto.nativetests/TestRepartitionQueriesWithSmallPages.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -27,10 +27,10 @@ public class TestRepartitionQueriesWithSmallPages
2727
@Override
2828
protected QueryRunner createQueryRunner() throws Exception
2929
{
30-
return PrestoNativeQueryRunnerUtils.createNativeQueryRunner(
30+
return PrestoNativeQueryRunnerUtils.createNativeQueryRunner(System.getProperty("storageFormat"),
3131
// Use small SerializedPages to force flushing
3232
ImmutableMap.of("driver.max-page-partitioning-buffer-size", "200B"),
33-
System.getProperty("storageFormat"));
33+
ImmutableMap.of());
3434
}
3535

3636
@Parameters("storageFormat")

‎presto-native-tests/src/test/java/com.facebook.presto.nativetests/TestWindowQueries.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,8 @@ public class TestWindowQueries
3030
@Override
3131
protected QueryRunner createQueryRunner() throws Exception
3232
{
33-
return PrestoNativeQueryRunnerUtils.createNativeQueryRunner(ImmutableMap.of(), System.getProperty("storageFormat"));
33+
return PrestoNativeQueryRunnerUtils.createNativeQueryRunner(System.getProperty("storageFormat"),
34+
ImmutableMap.of(), ImmutableMap.of());
3435
}
3536

3637
@Parameters("storageFormat")

0 commit comments

Comments
 (0)