Skip to content

Commit 54a233c

Browse files
HyukjinKwonHyukjinKwon
committed
Revert "Revert "[SPARK-28461][SQL] Pad Decimal numbers with trailing zeros to the scale of the column""
This reverts commit 9cd174a.
1 parent b36bc91 commit 54a233c

36 files changed

+847
-807
lines changed

‎docs/sql-migration-guide.md

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -226,6 +226,32 @@ license: |
226226

227227
- Since Spark 3.0, when casting string value to date, timestamp and interval values, the leading and trailing white spaces(<= ACSII 32) will be trimmed before casing, e.g. `cast('2019-10-10\t as date)` results the date value `2019-10-10`. In Spark version 2.4 and earlier, only the trailing space will be removed, thus, the result is `null`.
228228

229+
- Since Spark 3.0, we pad decimal numbers with trailing zeros to the scale of the column for `spark-sql` interface, for example:
230+
<table class="table">
231+
<tr>
232+
<th>
233+
<b>Query</b>
234+
</th>
235+
<th>
236+
<b>Spark 2.4 or Prior</b>
237+
</th>
238+
<th>
239+
<b>Spark 3.0</b>
240+
</th>
241+
</tr>
242+
<tr>
243+
<td>
244+
<code>SELECT CAST(1 AS decimal(38, 18));</code>
245+
</td>
246+
<td>
247+
<code>1</code>
248+
</td>
249+
<td>
250+
<code>1.000000000000000000</code>
251+
</td>
252+
</tr>
253+
</table>
254+
229255
## Upgrading from Spark SQL 2.4 to 2.4.1
230256

231257
- The value of `spark.executor.heartbeatInterval`, when specified without units like "30" rather than "30s", was

‎sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/ApproximatePercentile.scala

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -63,7 +63,7 @@ import org.apache.spark.sql.types._
6363
> SELECT _FUNC_(10.0, array(0.5, 0.4, 0.1), 100);
6464
[10.0,10.0,10.0]
6565
> SELECT _FUNC_(10.0, 0.5, 100);
66-
10
66+
10.0
6767
""",
6868
since = "2.1.0")
6969
case class ApproximatePercentile(

‎sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/conditionalExpressions.scala

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -111,9 +111,9 @@ case class If(predicate: Expression, trueValue: Expression, falseValue: Expressi
111111
examples = """
112112
Examples:
113113
> SELECT CASE WHEN 1 > 0 THEN 1 WHEN 2 > 0 THEN 2.0 ELSE 1.2 END;
114-
1
114+
1.0
115115
> SELECT CASE WHEN 1 < 0 THEN 1 WHEN 2 > 0 THEN 2.0 ELSE 1.2 END;
116-
2
116+
2.0
117117
> SELECT CASE WHEN 1 < 0 THEN 1 WHEN 2 < 0 THEN 2.0 END;
118118
NULL
119119
""")

‎sql/core/src/main/scala/org/apache/spark/sql/execution/HiveResult.scala

Lines changed: 1 addition & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -59,14 +59,6 @@ object HiveResult {
5959
result.map(_.zip(types).map(toHiveString)).map(_.mkString("\t"))
6060
}
6161

62-
private def formatDecimal(d: java.math.BigDecimal): String = {
63-
if (d.compareTo(java.math.BigDecimal.ZERO) == 0) {
64-
java.math.BigDecimal.ZERO.toPlainString
65-
} else {
66-
d.stripTrailingZeros().toPlainString // Hive strips trailing zeros
67-
}
68-
}
69-
7062
private val primitiveTypes = Seq(
7163
StringType,
7264
IntegerType,
@@ -127,7 +119,7 @@ object HiveResult {
127119
case (t: Timestamp, TimestampType) =>
128120
DateTimeUtils.timestampToString(timestampFormatter, DateTimeUtils.fromJavaTimestamp(t))
129121
case (bin: Array[Byte], BinaryType) => new String(bin, StandardCharsets.UTF_8)
130-
case (decimal: java.math.BigDecimal, DecimalType()) => formatDecimal(decimal)
122+
case (decimal: java.math.BigDecimal, DecimalType()) => decimal.toPlainString
131123
case (interval: CalendarInterval, CalendarIntervalType) =>
132124
SQLConf.get.intervalOutputStyle match {
133125
case SQL_STANDARD => toSqlStandardString(interval)

‎sql/core/src/test/resources/sql-tests/results/ansi/decimalArithmeticOperations.sql.out

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -24,50 +24,50 @@ select id, a*10, b/10 from decimals_test order by id
2424
-- !query 2 schema
2525
struct<id:int,(CAST(a AS DECIMAL(38,18)) * CAST(CAST(10 AS DECIMAL(2,0)) AS DECIMAL(38,18))):decimal(38,15),(CAST(b AS DECIMAL(38,18)) / CAST(CAST(10 AS DECIMAL(2,0)) AS DECIMAL(38,18))):decimal(38,18)>
2626
-- !query 2 output
27-
1 1000 99.9
28-
2 123451.23 1234.5123
29-
3 1.234567891011 123.41
30-
4 1234567891234567890 0.112345678912345679
27+
1 1000.000000000000000 99.900000000000000000
28+
2 123451.230000000000000 1234.512300000000000000
29+
3 1.234567891011000 123.410000000000000000
30+
4 1234567891234567890.000000000000000 0.112345678912345679
3131

3232

3333
-- !query 3
3434
select 10.3 * 3.0
3535
-- !query 3 schema
3636
struct<(CAST(10.3 AS DECIMAL(3,1)) * CAST(3.0 AS DECIMAL(3,1))):decimal(6,2)>
3737
-- !query 3 output
38-
30.9
38+
30.90
3939

4040

4141
-- !query 4
4242
select 10.3000 * 3.0
4343
-- !query 4 schema
4444
struct<(CAST(10.3000 AS DECIMAL(6,4)) * CAST(3.0 AS DECIMAL(6,4))):decimal(9,5)>
4545
-- !query 4 output
46-
30.9
46+
30.90000
4747

4848

4949
-- !query 5
5050
select 10.30000 * 30.0
5151
-- !query 5 schema
5252
struct<(CAST(10.30000 AS DECIMAL(7,5)) * CAST(30.0 AS DECIMAL(7,5))):decimal(11,6)>
5353
-- !query 5 output
54-
309
54+
309.000000
5555

5656

5757
-- !query 6
5858
select 10.300000000000000000 * 3.000000000000000000
5959
-- !query 6 schema
6060
struct<(CAST(10.300000000000000000 AS DECIMAL(20,18)) * CAST(3.000000000000000000 AS DECIMAL(20,18))):decimal(38,34)>
6161
-- !query 6 output
62-
30.9
62+
30.9000000000000000000000000000000000
6363

6464

6565
-- !query 7
6666
select 10.300000000000000000 * 3.0000000000000000000
6767
-- !query 7 schema
6868
struct<(CAST(10.300000000000000000 AS DECIMAL(21,19)) * CAST(3.0000000000000000000 AS DECIMAL(21,19))):decimal(38,34)>
6969
-- !query 7 output
70-
30.9
70+
30.9000000000000000000000000000000000
7171

7272

7373
-- !query 8

‎sql/core/src/test/resources/sql-tests/results/ansi/literals.sql.out

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -169,7 +169,7 @@ select 1D, 1.2D, 1e10, 1.5e5, .10D, 0.10D, .1e5, .9e+2, 0.9e+2, 900e-1, 9.e+1
169169
-- !query 17 schema
170170
struct<1.0:double,1.2:double,1E+10:decimal(1,-10),1.5E+5:decimal(2,-4),0.1:double,0.1:double,1E+4:decimal(1,-4),9E+1:decimal(1,-1),9E+1:decimal(1,-1),90.0:decimal(3,1),9E+1:decimal(1,-1)>
171171
-- !query 17 output
172-
1.0 1.2 10000000000 150000 0.1 0.1 10000 90 90 90 90
172+
1.0 1.2 10000000000 150000 0.1 0.1 10000 90 90 90.0 90
173173

174174

175175
-- !query 18
@@ -339,7 +339,7 @@ select 90912830918230182310293801923652346786BD, 123.0E-28BD, 123.08BD
339339
-- !query 35 schema
340340
struct<90912830918230182310293801923652346786:decimal(38,0),1.230E-26:decimal(29,29),123.08:decimal(5,2)>
341341
-- !query 35 output
342-
90912830918230182310293801923652346786 0.0000000000000000000000000123 123.08
342+
90912830918230182310293801923652346786 0.00000000000000000000000001230 123.08
343343

344344

345345
-- !query 36

‎sql/core/src/test/resources/sql-tests/results/decimalArithmeticOperations.sql.out

Lines changed: 23 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -56,9 +56,9 @@ select id, a+b, a-b, a*b, a/b from decimals_test order by id
5656
-- !query 6 schema
5757
struct<id:int,(a + b):decimal(38,17),(a - b):decimal(38,17),(a * b):decimal(38,6),(a / b):decimal(38,6)>
5858
-- !query 6 output
59-
1 1099 -899 99900 0.1001
60-
2 24690.246 0 152402061.885129 1
61-
3 1234.2234567891011 -1233.9765432108989 152.358023 0.0001
59+
1 1099.00000000000000000 -899.00000000000000000 99900.000000 0.100100
60+
2 24690.24600000000000000 0.00000000000000000 152402061.885129 1.000000
61+
3 1234.22345678910110000 -1233.97654321089890000 152.358023 0.000100
6262
4 123456789123456790.12345678912345679 123456789123456787.87654321087654321 138698367904130467.515623 109890109097814272.043109
6363

6464

@@ -67,50 +67,50 @@ select id, a*10, b/10 from decimals_test order by id
6767
-- !query 7 schema
6868
struct<id:int,(CAST(a AS DECIMAL(38,18)) * CAST(CAST(10 AS DECIMAL(2,0)) AS DECIMAL(38,18))):decimal(38,15),(CAST(b AS DECIMAL(38,18)) / CAST(CAST(10 AS DECIMAL(2,0)) AS DECIMAL(38,18))):decimal(38,18)>
6969
-- !query 7 output
70-
1 1000 99.9
71-
2 123451.23 1234.5123
72-
3 1.234567891011 123.41
73-
4 1234567891234567890 0.112345678912345679
70+
1 1000.000000000000000 99.900000000000000000
71+
2 123451.230000000000000 1234.512300000000000000
72+
3 1.234567891011000 123.410000000000000000
73+
4 1234567891234567890.000000000000000 0.112345678912345679
7474

7575

7676
-- !query 8
7777
select 10.3 * 3.0
7878
-- !query 8 schema
7979
struct<(CAST(10.3 AS DECIMAL(3,1)) * CAST(3.0 AS DECIMAL(3,1))):decimal(6,2)>
8080
-- !query 8 output
81-
30.9
81+
30.90
8282

8383

8484
-- !query 9
8585
select 10.3000 * 3.0
8686
-- !query 9 schema
8787
struct<(CAST(10.3000 AS DECIMAL(6,4)) * CAST(3.0 AS DECIMAL(6,4))):decimal(9,5)>
8888
-- !query 9 output
89-
30.9
89+
30.90000
9090

9191

9292
-- !query 10
9393
select 10.30000 * 30.0
9494
-- !query 10 schema
9595
struct<(CAST(10.30000 AS DECIMAL(7,5)) * CAST(30.0 AS DECIMAL(7,5))):decimal(11,6)>
9696
-- !query 10 output
97-
309
97+
309.000000
9898

9999

100100
-- !query 11
101101
select 10.300000000000000000 * 3.000000000000000000
102102
-- !query 11 schema
103103
struct<(CAST(10.300000000000000000 AS DECIMAL(20,18)) * CAST(3.000000000000000000 AS DECIMAL(20,18))):decimal(38,34)>
104104
-- !query 11 output
105-
30.9
105+
30.9000000000000000000000000000000000
106106

107107

108108
-- !query 12
109109
select 10.300000000000000000 * 3.0000000000000000000
110110
-- !query 12 schema
111111
struct<(CAST(10.300000000000000000 AS DECIMAL(21,19)) * CAST(3.0000000000000000000 AS DECIMAL(21,19))):decimal(38,34)>
112112
-- !query 12 output
113-
30.9
113+
30.9000000000000000000000000000000000
114114

115115

116116
-- !query 13
@@ -198,9 +198,9 @@ select id, a+b, a-b, a*b, a/b from decimals_test order by id
198198
-- !query 23 schema
199199
struct<id:int,(a + b):decimal(38,18),(a - b):decimal(38,18),(a * b):decimal(38,36),(a / b):decimal(38,18)>
200200
-- !query 23 output
201-
1 1099 -899 NULL 0.1001001001001001
202-
2 24690.246 0 NULL 1
203-
3 1234.2234567891011 -1233.9765432108989 NULL 0.000100037913541123
201+
1 1099.000000000000000000 -899.000000000000000000 NULL 0.100100100100100100
202+
2 24690.246000000000000000 0.000000000000000000 NULL 1.000000000000000000
203+
3 1234.223456789101100000 -1233.976543210898900000 NULL 0.000100037913541123
204204
4 123456789123456790.123456789123456789 123456789123456787.876543210876543211 NULL 109890109097814272.043109406191131436
205205

206206

@@ -209,42 +209,42 @@ select id, a*10, b/10 from decimals_test order by id
209209
-- !query 24 schema
210210
struct<id:int,(CAST(a AS DECIMAL(38,18)) * CAST(CAST(10 AS DECIMAL(2,0)) AS DECIMAL(38,18))):decimal(38,18),(CAST(b AS DECIMAL(38,18)) / CAST(CAST(10 AS DECIMAL(2,0)) AS DECIMAL(38,18))):decimal(38,19)>
211211
-- !query 24 output
212-
1 1000 99.9
213-
2 123451.23 1234.5123
214-
3 1.234567891011 123.41
215-
4 1234567891234567890 0.1123456789123456789
212+
1 1000.000000000000000000 99.9000000000000000000
213+
2 123451.230000000000000000 1234.5123000000000000000
214+
3 1.234567891011000000 123.4100000000000000000
215+
4 1234567891234567890.000000000000000000 0.1123456789123456789
216216

217217

218218
-- !query 25
219219
select 10.3 * 3.0
220220
-- !query 25 schema
221221
struct<(CAST(10.3 AS DECIMAL(3,1)) * CAST(3.0 AS DECIMAL(3,1))):decimal(6,2)>
222222
-- !query 25 output
223-
30.9
223+
30.90
224224

225225

226226
-- !query 26
227227
select 10.3000 * 3.0
228228
-- !query 26 schema
229229
struct<(CAST(10.3000 AS DECIMAL(6,4)) * CAST(3.0 AS DECIMAL(6,4))):decimal(9,5)>
230230
-- !query 26 output
231-
30.9
231+
30.90000
232232

233233

234234
-- !query 27
235235
select 10.30000 * 30.0
236236
-- !query 27 schema
237237
struct<(CAST(10.30000 AS DECIMAL(7,5)) * CAST(30.0 AS DECIMAL(7,5))):decimal(11,6)>
238238
-- !query 27 output
239-
309
239+
309.000000
240240

241241

242242
-- !query 28
243243
select 10.300000000000000000 * 3.000000000000000000
244244
-- !query 28 schema
245245
struct<(CAST(10.300000000000000000 AS DECIMAL(20,18)) * CAST(3.000000000000000000 AS DECIMAL(20,18))):decimal(38,36)>
246246
-- !query 28 output
247-
30.9
247+
30.900000000000000000000000000000000000
248248

249249

250250
-- !query 29

‎sql/core/src/test/resources/sql-tests/results/literals.sql.out

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -169,7 +169,7 @@ select 1D, 1.2D, 1e10, 1.5e5, .10D, 0.10D, .1e5, .9e+2, 0.9e+2, 900e-1, 9.e+1
169169
-- !query 17 schema
170170
struct<1.0:double,1.2:double,1E+10:decimal(1,-10),1.5E+5:decimal(2,-4),0.1:double,0.1:double,1E+4:decimal(1,-4),9E+1:decimal(1,-1),9E+1:decimal(1,-1),90.0:decimal(3,1),9E+1:decimal(1,-1)>
171171
-- !query 17 output
172-
1.0 1.2 10000000000 150000 0.1 0.1 10000 90 90 90 90
172+
1.0 1.2 10000000000 150000 0.1 0.1 10000 90 90 90.0 90
173173

174174

175175
-- !query 18
@@ -339,7 +339,7 @@ select 90912830918230182310293801923652346786BD, 123.0E-28BD, 123.08BD
339339
-- !query 35 schema
340340
struct<90912830918230182310293801923652346786:decimal(38,0),1.230E-26:decimal(29,29),123.08:decimal(5,2)>
341341
-- !query 35 output
342-
90912830918230182310293801923652346786 0.0000000000000000000000000123 123.08
342+
90912830918230182310293801923652346786 0.00000000000000000000000001230 123.08
343343

344344

345345
-- !query 36

‎sql/core/src/test/resources/sql-tests/results/order-by-nulls-ordering.sql.out

Lines changed: 24 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -206,44 +206,44 @@ select * from spark_10747_mix order by col1 nulls last, col5 nulls last
206206
-- !query 13 schema
207207
struct<col1:string,col2:int,col3:double,col4:decimal(10,2),col5:decimal(20,1)>
208208
-- !query 13 output
209-
a 1 1.0 1 NULL
210-
b 2 1.0 1 10
211-
c 3 2.0 2 15.1
212-
c 3 2.0 2 NULL
213-
d 3 2.0 3 0
214-
d 3 0.0 3 1
215-
d 3 NULL 4 1
216-
NULL 3 0.0 3 1
209+
a 1 1.0 1.00 NULL
210+
b 2 1.0 1.00 10.0
211+
c 3 2.0 2.00 15.1
212+
c 3 2.0 2.00 NULL
213+
d 3 2.0 3.00 0.0
214+
d 3 0.0 3.00 1.0
215+
d 3 NULL 4.00 1.0
216+
NULL 3 0.0 3.00 1.0
217217

218218

219219
-- !query 14
220220
select * from spark_10747_mix order by col1 desc nulls first, col5 desc nulls first
221221
-- !query 14 schema
222222
struct<col1:string,col2:int,col3:double,col4:decimal(10,2),col5:decimal(20,1)>
223223
-- !query 14 output
224-
NULL 3 0.0 3 1
225-
d 3 0.0 3 1
226-
d 3 NULL 4 1
227-
d 3 2.0 3 0
228-
c 3 2.0 2 NULL
229-
c 3 2.0 2 15.1
230-
b 2 1.0 1 10
231-
a 1 1.0 1 NULL
224+
NULL 3 0.0 3.00 1.0
225+
d 3 0.0 3.00 1.0
226+
d 3 NULL 4.00 1.0
227+
d 3 2.0 3.00 0.0
228+
c 3 2.0 2.00 NULL
229+
c 3 2.0 2.00 15.1
230+
b 2 1.0 1.00 10.0
231+
a 1 1.0 1.00 NULL
232232

233233

234234
-- !query 15
235235
select * from spark_10747_mix order by col5 desc nulls first, col3 desc nulls last
236236
-- !query 15 schema
237237
struct<col1:string,col2:int,col3:double,col4:decimal(10,2),col5:decimal(20,1)>
238238
-- !query 15 output
239-
c 3 2.0 2 NULL
240-
a 1 1.0 1 NULL
241-
c 3 2.0 2 15.1
242-
b 2 1.0 1 10
243-
d 3 0.0 3 1
244-
NULL 3 0.0 3 1
245-
d 3 NULL 4 1
246-
d 3 2.0 3 0
239+
c 3 2.0 2.00 NULL
240+
a 1 1.0 1.00 NULL
241+
c 3 2.0 2.00 15.1
242+
b 2 1.0 1.00 10.0
243+
d 3 0.0 3.00 1.0
244+
NULL 3 0.0 3.00 1.0
245+
d 3 NULL 4.00 1.0
246+
d 3 2.0 3.00 0.0
247247

248248

249249
-- !query 16

‎sql/core/src/test/resources/sql-tests/results/postgreSQL/create_view.sql.out

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -956,7 +956,7 @@ SELECT * FROM tt1
956956
struct<a:string,b:string,c:decimal(12,2),d:string>
957957
-- !query 83 output
958958
0123456789 abc 42.12 abc
959-
abc 0123456789 42 abcd
959+
abc 0123456789 42.00 abcd
960960

961961

962962
-- !query 84

0 commit comments

Comments
 (0)