Skip to content

Commit c43b45f

Browse files
author
jlf
authored
KE-43300 use crontab expression to periodicGC in KE (apache#724)
* KE-43300 use crontab expression to periodicGC in KE * KE-43300 fix review
1 parent faf3d33 commit c43b45f

File tree

2 files changed

+12
-2
lines changed

2 files changed

+12
-2
lines changed

core/src/main/scala/org/apache/spark/ContextCleaner.scala

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -91,6 +91,8 @@ private[spark] class ContextCleaner(
9191
*/
9292
private val periodicGCInterval = sc.conf.get(CLEANER_PERIODIC_GC_INTERVAL)
9393

94+
private val periodicGCEnabled = sc.conf.get(CLEANER_PERIODIC_GC_ENABLED)
95+
9496
/**
9597
* Whether the cleaning thread will block on cleanup tasks (other than shuffle, which
9698
* is controlled by the `spark.cleaner.referenceTracking.blocking.shuffle` parameter).
@@ -137,8 +139,10 @@ private[spark] class ContextCleaner(
137139
cleaningThread.setDaemon(true)
138140
cleaningThread.setName("Spark Context Cleaner")
139141
cleaningThread.start()
140-
periodicGCService.scheduleAtFixedRate(() => System.gc(),
141-
periodicGCInterval, periodicGCInterval, TimeUnit.SECONDS)
142+
if (periodicGCEnabled) {
143+
periodicGCService.scheduleAtFixedRate(() => System.gc(),
144+
periodicGCInterval, periodicGCInterval, TimeUnit.SECONDS)
145+
}
142146
}
143147

144148
/**

core/src/main/scala/org/apache/spark/internal/config/package.scala

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1709,6 +1709,12 @@ package object config {
17091709
.timeConf(TimeUnit.SECONDS)
17101710
.createWithDefaultString("30min")
17111711

1712+
private[spark] val CLEANER_PERIODIC_GC_ENABLED =
1713+
ConfigBuilder("spark.cleaner.periodicGC.enabled")
1714+
.version("3.3.0")
1715+
.booleanConf
1716+
.createWithDefault(true)
1717+
17121718
private[spark] val CLEANER_REFERENCE_TRACKING =
17131719
ConfigBuilder("spark.cleaner.referenceTracking")
17141720
.version("1.0.0")

0 commit comments

Comments
 (0)