Skip to content

Commit 2b48831

Browse files
small changes with sean owen's comments
1 parent 2973024 commit 2b48831

File tree

2 files changed

+8
-7
lines changed

2 files changed

+8
-7
lines changed

core/src/main/scala/org/apache/spark/deploy/master/Master.scala

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -764,9 +764,10 @@ private[spark] class Master(
764764
val replayBus = new ReplayListenerBus()
765765
val ui = SparkUI.createHistoryUI(new SparkConf, replayBus, new SecurityManager(conf),
766766
appName + status, HistoryServer.UI_PATH_PREFIX + s"/${app.id}")
767+
val maybeTruncated = eventLogFile.endsWith(EventLoggingListener.IN_PROGRESS)
767768
try {
768769
replayBus.replay(logInput, eventLogFile,
769-
eventLogFile.endsWith(EventLoggingListener.IN_PROGRESS))
770+
maybeTruncated)
770771
} finally {
771772
logInput.close()
772773
}

core/src/main/scala/org/apache/spark/scheduler/ReplayListenerBus.scala

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -41,13 +41,13 @@ private[spark] class ReplayListenerBus extends SparkListenerBus with Logging {
4141
*
4242
* @param logData Stream containing event log data.
4343
* @param sourceName Filename (or other source identifier) from whence @logData is being read
44-
* @param sourceTruncated Indicate whether log file might be truncated (some abnormal situation
45-
* encountered, log file not finish writing) or not
44+
* @param maybeTruncated Indicate whether log file might be truncated (some abnormal situations
45+
* encountered, log file might not finished writing) or not
4646
*/
4747
def replay(
4848
logData: InputStream,
4949
sourceName: String,
50-
sourceTruncated: Boolean = false): Unit = {
50+
maybeTruncated: Boolean = false): Unit = {
5151
var currentLine: String = null
5252
var lineNumber: Int = 1
5353
try {
@@ -59,11 +59,11 @@ private[spark] class ReplayListenerBus extends SparkListenerBus with Logging {
5959
} catch {
6060
case jpe: JsonParseException =>
6161
// We can only ignore exception from last line of the file that might be truncated
62-
if (!sourceTruncated || lines.hasNext) {
62+
if (!maybeTruncated || lines.hasNext) {
6363
throw jpe
6464
} else {
65-
logWarning(s"Get json parse exception from log file $sourceName" +
66-
s" in line $lineNumber, the file might not finished writing.")
65+
logWarning(s"Got JsonParseException from log file $sourceName" +
66+
s" at line $lineNumber, the file might not have finished writing cleanly.")
6767
}
6868
}
6969
lineNumber += 1

0 commit comments

Comments
 (0)