Skip to content

Commit 480bd1d

Browse files
committed
[SPARK-4908][SQL] Prevent multiple concurrent hive native commands
This is just a quick fix that locks when calling `runHive`. If we can find a way to avoid the error without a global lock that would be better. Author: Michael Armbrust <[email protected]> Closes #3834 from marmbrus/hiveConcurrency and squashes the following commits: bf25300 [Michael Armbrust] prevent multiple concurrent hive native commands
1 parent efa80a5 commit 480bd1d

File tree

2 files changed

+8
-1
lines changed

2 files changed

+8
-1
lines changed

sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveContext.scala

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -284,7 +284,7 @@ class HiveContext(sc: SparkContext) extends SQLContext(sc) {
284284
* Execute the command using Hive and return the results as a sequence. Each element
285285
* in the sequence is one row.
286286
*/
287-
protected def runHive(cmd: String, maxRows: Int = 1000): Seq[String] = {
287+
protected def runHive(cmd: String, maxRows: Int = 1000): Seq[String] = synchronized {
288288
try {
289289
val cmd_trimmed: String = cmd.trim()
290290
val tokens: Array[String] = cmd_trimmed.split("\\s+")

sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,13 @@ class HiveQuerySuite extends HiveComparisonTest with BeforeAndAfter {
5656
Locale.setDefault(originalLocale)
5757
}
5858

59+
test("SPARK-4908: concurent hive native commands") {
60+
(1 to 100).par.map { _ =>
61+
sql("USE default")
62+
sql("SHOW TABLES")
63+
}
64+
}
65+
5966
createQueryTest("constant object inspector for generic udf",
6067
"""SELECT named_struct(
6168
lower("AA"), "10",

0 commit comments

Comments
 (0)