[SC-6106] Atomic commit protocol unit test hangs in HashMap.elemEquals

## What changes were proposed in this pull request?

We saw the following hang during a jenkins run:

```
"Executor task launch worker for task 281" apache#24085 daemon prio=5 os_prio=0 tid=0x00007fa140696000 nid=0x78fc runnable [0x00007fa06e7f7000]
   java.lang.Thread.State: RUNNABLE
    at java.net.URI.equals(URI.java:1421)
    at org.apache.hadoop.fs.Path.equals(Path.java:392)
    at scala.collection.mutable.HashTable$class.elemEquals(HashTable.scala:358)
    at scala.collection.mutable.HashMap.elemEquals(HashMap.scala:40)
    at scala.collection.mutable.HashTable$class.scala$collection$mutable$HashTable$$findEntry0(HashTable.scala:136)
    at scala.collection.mutable.HashTable$class.findOrAddEntry(HashTable.scala:165)
    at scala.collection.mutable.HashMap.findOrAddEntry(HashMap.scala:40)
    at scala.collection.mutable.HashMap.put(HashMap.scala:76)
    at scala.collection.mutable.HashMap.update(HashMap.scala:81)
    at org.apache.spark.sql.transaction.DatabricksAtomicCommitProtocolSuite$FakeClockFileSystem.create(DatabricksAtomicCommitProtocolSuite.scala:355)
```

Since URI.equals is non-blocking and the thread is at 100% cpu, the hash map must have gotten a loop in it due to concurrent accesses. The fix should be to synchronize on accesses to that map during tests.

## How was this patch tested?

Ran existing tests.

Author: Eric Liang <[email protected]>

Closes apache#269 from ericl/sc-6106.

Author: ericl

sql/core/src/test/scala/com/databricks/sql/transaction/DatabricksAtomicCommitProtocolSuite.scala

@@ -253,9 +253,9 @@ class DatabricksAtomicCommitProtocolSuite extends QueryTest with SharedSQLContex
 
   test("reader re-lists directory when files may be missing from the initial list") {
     withTempDir { dir =>
-      var listCount = 0
+      @volatile var listCount = 0
       val fs = new RawLocalFileSystem() {
-        override def listStatus(path: Path): Array[FileStatus] = {
+        override def listStatus(path: Path): Array[FileStatus] = synchronized {
           listCount += 1
           super.listStatus(path)
         }
@@ -311,9 +311,9 @@ class DatabricksAtomicCommitProtocolSuite extends QueryTest with SharedSQLContex
 
   test("re-list is avoided after a grace period") {
     withTempDir { dir =>
-      var listCount = 0
+      @volatile var listCount = 0
       val fs = new RawLocalFileSystem() {
-        override def listStatus(path: Path): Array[FileStatus] = {
+        override def listStatus(path: Path): Array[FileStatus] = synchronized {
           listCount += 1
           super.listStatus(path)
         }
@@ -351,13 +351,13 @@ class DatabricksAtomicCommitProtocolSuite extends QueryTest with SharedSQLContex
 
     // override file creation to save our custom timestamps
     override def create(path: Path): FSDataOutputStream = create(path, false)
-    override def create(path: Path, overwrite: Boolean): FSDataOutputStream = {
+    override def create(path: Path, overwrite: Boolean): FSDataOutputStream = synchronized {
       creationTimes(path.makeQualified(this)) = clock.getTimeMillis()
       super.create(path, overwrite)
     }
 
     // fill in our custom timestamps on list
-    override def listStatus(path: Path): Array[FileStatus] = {
+    override def listStatus(path: Path): Array[FileStatus] = synchronized {
       super.listStatus(path).map { stat =>
         new FileStatus(
           stat.getLen,
@@ -589,10 +589,10 @@ class DatabricksAtomicCommitProtocolSuite extends QueryTest with SharedSQLContex
      * Emulates S3 list consistency guarantees. We assume read-after-write for single keys,
      * however a list call is not atomic and so may observe writes out of order.
      */
-    var numLists = 0
+    @volatile var numLists = 0
     val inconsistentFs = new RawLocalFileSystem() {
       val consistentFiles = mutable.Set[Path]()
-      override def listStatus(path: Path): Array[FileStatus] = {
+      override def listStatus(path: Path): Array[FileStatus] = synchronized {
         numLists += 1
         super.listStatus(path).filter { stat =>
           stat.getPath match {
@@ -648,7 +648,7 @@ class DatabricksAtomicCommitProtocolSuite extends QueryTest with SharedSQLContex
     val clock = new ManualClock(123456789L)
 
     val fakeFs = new RawLocalFileSystem() {
-      override def listStatus(path: Path): Array[FileStatus] = {
+      override def listStatus(path: Path): Array[FileStatus] = synchronized {
         super.listStatus(path).map { stat =>
           new FileStatus(
             stat.getLen,