diff --git a/build.gradle b/build.gradle index 267679356..336a956e2 100644 --- a/build.gradle +++ b/build.gradle @@ -130,8 +130,9 @@ dependencies { compile "org.jetbrains.kotlin:kotlin-stdlib-common:${kotlin_version}" compile 'org.jetbrains.kotlinx:kotlinx-coroutines-core:1.3.7' compile "org.jetbrains:annotations:13.0" - compile "com.amazon.opendistroforelasticsearch:notification:${opendistroVersion}.0" + compile "com.amazon.opendistroforelasticsearch:notification:1.13.1.0" compile "com.amazon.opendistroforelasticsearch:common-utils:${opendistroVersion}.0" + compile "com.github.seancfoley:ipaddress:5.3.3" testCompile "org.elasticsearch.test:framework:${es_version}" testCompile "org.jetbrains.kotlin:kotlin-test:${kotlin_version}" @@ -175,6 +176,7 @@ afterEvaluate { def firstPlugin = plugins.get(0) plugins.remove(0) plugins.add(firstPlugin) + if (securityEnabled) { node.extraConfigFile("kirk.pem", file("src/test/resources/security/kirk.pem")) node.extraConfigFile("kirk-key.pem", file("src/test/resources/security/kirk-key.pem")) diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/ManagedIndexRunner.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/ManagedIndexRunner.kt index 9e4e6a431..d0402109f 100644 --- a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/ManagedIndexRunner.kt +++ b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/ManagedIndexRunner.kt @@ -123,6 +123,7 @@ object ManagedIndexRunner : ScheduledJobRunner, private val errorNotificationRetryPolicy = BackoffPolicy.exponentialBackoff(TimeValue.timeValueMillis(250), 3) private var jobInterval: Int = DEFAULT_JOB_INTERVAL private var allowList: List = ALLOW_LIST_NONE + private var hostDenyList: List = emptyList() fun registerClusterService(clusterService: ClusterService): ManagedIndexRunner { this.clusterService = clusterService @@ -165,6 +166,8 @@ object ManagedIndexRunner : ScheduledJobRunner, clusterService.clusterSettings.addSettingsUpdateConsumer(ALLOW_LIST) { allowList = it } + + hostDenyList = settings.getAsList(ManagedIndexSettings.HOST_DENY_LIST) return this } @@ -225,7 +228,7 @@ object ManagedIndexRunner : ScheduledJobRunner, } val state = policy.getStateToExecute(managedIndexMetaData) - val action: Action? = state?.getActionToExecute(clusterService, scriptService, client, managedIndexMetaData) + val action: Action? = state?.getActionToExecute(clusterService, scriptService, client, settings, managedIndexMetaData) val step: Step? = action?.getStepToExecute() val currentActionMetaData = action?.getUpdatedActionMetaData(managedIndexMetaData, state) @@ -651,7 +654,7 @@ object ManagedIndexRunner : ScheduledJobRunner, policy.errorNotification?.run { errorNotificationRetryPolicy.retry(logger) { withContext(Dispatchers.IO) { - destination.publish(null, compileTemplate(messageTemplate, managedIndexMetaData)) + destination.publish(null, compileTemplate(messageTemplate, managedIndexMetaData), hostDenyList) } } } diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/action/NotificationAction.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/action/NotificationAction.kt index 9d73bd698..018e2829c 100644 --- a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/action/NotificationAction.kt +++ b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/action/NotificationAction.kt @@ -22,17 +22,19 @@ import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagemen import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.step.notification.AttemptNotificationStep import org.elasticsearch.client.Client import org.elasticsearch.cluster.service.ClusterService +import org.elasticsearch.common.settings.Settings import org.elasticsearch.script.ScriptService class NotificationAction( clusterService: ClusterService, scriptService: ScriptService, client: Client, + settings: Settings, managedIndexMetaData: ManagedIndexMetaData, config: NotificationActionConfig ) : Action(ActionConfig.ActionType.NOTIFICATION, config, managedIndexMetaData) { - private val attemptNotificationStep = AttemptNotificationStep(clusterService, scriptService, client, config, managedIndexMetaData) + private val attemptNotificationStep = AttemptNotificationStep(clusterService, scriptService, client, settings, config, managedIndexMetaData) private val steps = listOf(attemptNotificationStep) override fun getSteps(): List = steps diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/model/action/ActionConfig.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/model/action/ActionConfig.kt index 64ed6d300..83f355202 100644 --- a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/model/action/ActionConfig.kt +++ b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/model/action/ActionConfig.kt @@ -22,6 +22,7 @@ import org.elasticsearch.cluster.service.ClusterService import org.elasticsearch.common.io.stream.StreamInput import org.elasticsearch.common.io.stream.StreamOutput import org.elasticsearch.common.io.stream.Writeable +import org.elasticsearch.common.settings.Settings import org.elasticsearch.common.xcontent.ToXContent import org.elasticsearch.common.xcontent.ToXContentFragment import org.elasticsearch.common.xcontent.XContentBuilder @@ -51,6 +52,7 @@ abstract class ActionConfig( clusterService: ClusterService, scriptService: ScriptService, client: Client, + settings: Settings, managedIndexMetaData: ManagedIndexMetaData ): Action diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/model/action/AllocationActionConfig.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/model/action/AllocationActionConfig.kt index 67e5ca3ef..eaa2a9086 100644 --- a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/model/action/AllocationActionConfig.kt +++ b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/model/action/AllocationActionConfig.kt @@ -22,6 +22,7 @@ import org.elasticsearch.client.Client import org.elasticsearch.cluster.service.ClusterService import org.elasticsearch.common.io.stream.StreamInput import org.elasticsearch.common.io.stream.StreamOutput +import org.elasticsearch.common.settings.Settings import org.elasticsearch.common.xcontent.ToXContent import org.elasticsearch.common.xcontent.ToXContentObject import org.elasticsearch.common.xcontent.XContentBuilder @@ -49,6 +50,7 @@ data class AllocationActionConfig( clusterService: ClusterService, scriptService: ScriptService, client: Client, + settings: Settings, managedIndexMetaData: ManagedIndexMetaData ): Action = AllocationAction(clusterService, client, managedIndexMetaData, this) diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/model/action/CloseActionConfig.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/model/action/CloseActionConfig.kt index df3995adc..4a9705939 100644 --- a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/model/action/CloseActionConfig.kt +++ b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/model/action/CloseActionConfig.kt @@ -20,6 +20,7 @@ import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagemen import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.model.ManagedIndexMetaData import org.elasticsearch.client.Client import org.elasticsearch.cluster.service.ClusterService +import org.elasticsearch.common.settings.Settings import org.elasticsearch.common.xcontent.ToXContent import org.elasticsearch.common.xcontent.ToXContentObject import org.elasticsearch.common.xcontent.XContentBuilder @@ -46,6 +47,7 @@ data class CloseActionConfig( clusterService: ClusterService, scriptService: ScriptService, client: Client, + settings: Settings, managedIndexMetaData: ManagedIndexMetaData ): Action = CloseAction(clusterService, client, managedIndexMetaData, this) diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/model/action/DeleteActionConfig.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/model/action/DeleteActionConfig.kt index ba4b6a68d..1088d617e 100644 --- a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/model/action/DeleteActionConfig.kt +++ b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/model/action/DeleteActionConfig.kt @@ -20,6 +20,7 @@ import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagemen import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.model.ManagedIndexMetaData import org.elasticsearch.client.Client import org.elasticsearch.cluster.service.ClusterService +import org.elasticsearch.common.settings.Settings import org.elasticsearch.common.xcontent.ToXContent import org.elasticsearch.common.xcontent.ToXContentObject import org.elasticsearch.common.xcontent.XContentBuilder @@ -46,6 +47,7 @@ data class DeleteActionConfig( clusterService: ClusterService, scriptService: ScriptService, client: Client, + settings: Settings, managedIndexMetaData: ManagedIndexMetaData ): Action = DeleteAction(clusterService, client, managedIndexMetaData, this) diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/model/action/ForceMergeActionConfig.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/model/action/ForceMergeActionConfig.kt index de41bb659..61e497152 100644 --- a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/model/action/ForceMergeActionConfig.kt +++ b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/model/action/ForceMergeActionConfig.kt @@ -22,6 +22,7 @@ import org.elasticsearch.client.Client import org.elasticsearch.cluster.service.ClusterService import org.elasticsearch.common.io.stream.StreamInput import org.elasticsearch.common.io.stream.StreamOutput +import org.elasticsearch.common.settings.Settings import org.elasticsearch.common.xcontent.ToXContent import org.elasticsearch.common.xcontent.ToXContentObject import org.elasticsearch.common.xcontent.XContentBuilder @@ -55,6 +56,7 @@ data class ForceMergeActionConfig( clusterService: ClusterService, scriptService: ScriptService, client: Client, + settings: Settings, managedIndexMetaData: ManagedIndexMetaData ): Action = ForceMergeAction(clusterService, client, managedIndexMetaData, this) diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/model/action/IndexPriorityActionConfig.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/model/action/IndexPriorityActionConfig.kt index a4f479eb0..305a309c2 100644 --- a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/model/action/IndexPriorityActionConfig.kt +++ b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/model/action/IndexPriorityActionConfig.kt @@ -22,6 +22,7 @@ import org.elasticsearch.client.Client import org.elasticsearch.cluster.service.ClusterService import org.elasticsearch.common.io.stream.StreamInput import org.elasticsearch.common.io.stream.StreamOutput +import org.elasticsearch.common.settings.Settings import org.elasticsearch.common.xcontent.ToXContent import org.elasticsearch.common.xcontent.ToXContentObject import org.elasticsearch.common.xcontent.XContentBuilder @@ -53,6 +54,7 @@ data class IndexPriorityActionConfig( clusterService: ClusterService, scriptService: ScriptService, client: Client, + settings: Settings, managedIndexMetaData: ManagedIndexMetaData ): Action = IndexPriorityAction(clusterService, client, managedIndexMetaData, this) diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/model/action/NotificationActionConfig.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/model/action/NotificationActionConfig.kt index 369c7b307..c820770cf 100644 --- a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/model/action/NotificationActionConfig.kt +++ b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/model/action/NotificationActionConfig.kt @@ -23,6 +23,7 @@ import org.elasticsearch.client.Client import org.elasticsearch.cluster.service.ClusterService import org.elasticsearch.common.io.stream.StreamInput import org.elasticsearch.common.io.stream.StreamOutput +import org.elasticsearch.common.settings.Settings import org.elasticsearch.common.xcontent.ToXContent import org.elasticsearch.common.xcontent.ToXContentObject import org.elasticsearch.common.xcontent.XContentBuilder @@ -59,8 +60,9 @@ data class NotificationActionConfig( clusterService: ClusterService, scriptService: ScriptService, client: Client, + settings: Settings, managedIndexMetaData: ManagedIndexMetaData - ): Action = NotificationAction(clusterService, scriptService, client, managedIndexMetaData, this) + ): Action = NotificationAction(clusterService, scriptService, client, settings, managedIndexMetaData, this) @Throws(IOException::class) constructor(sin: StreamInput) : this( diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/model/action/OpenActionConfig.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/model/action/OpenActionConfig.kt index 4833d7a31..c49d964f1 100644 --- a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/model/action/OpenActionConfig.kt +++ b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/model/action/OpenActionConfig.kt @@ -20,6 +20,7 @@ import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagemen import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.model.ManagedIndexMetaData import org.elasticsearch.client.Client import org.elasticsearch.cluster.service.ClusterService +import org.elasticsearch.common.settings.Settings import org.elasticsearch.common.xcontent.ToXContent import org.elasticsearch.common.xcontent.ToXContentObject import org.elasticsearch.common.xcontent.XContentBuilder @@ -46,6 +47,7 @@ data class OpenActionConfig( clusterService: ClusterService, scriptService: ScriptService, client: Client, + settings: Settings, managedIndexMetaData: ManagedIndexMetaData ): Action = OpenAction(clusterService, client, managedIndexMetaData, this) diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/model/action/ReadOnlyActionConfig.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/model/action/ReadOnlyActionConfig.kt index b01291ef8..0f69a5d1f 100644 --- a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/model/action/ReadOnlyActionConfig.kt +++ b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/model/action/ReadOnlyActionConfig.kt @@ -20,6 +20,7 @@ import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagemen import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.model.ManagedIndexMetaData import org.elasticsearch.client.Client import org.elasticsearch.cluster.service.ClusterService +import org.elasticsearch.common.settings.Settings import org.elasticsearch.common.xcontent.ToXContent import org.elasticsearch.common.xcontent.ToXContentObject import org.elasticsearch.common.xcontent.XContentBuilder @@ -46,6 +47,7 @@ data class ReadOnlyActionConfig( clusterService: ClusterService, scriptService: ScriptService, client: Client, + settings: Settings, managedIndexMetaData: ManagedIndexMetaData ): Action = ReadOnlyAction(clusterService, client, managedIndexMetaData, this) diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/model/action/ReadWriteActionConfig.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/model/action/ReadWriteActionConfig.kt index e445ddcc2..957f03e41 100644 --- a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/model/action/ReadWriteActionConfig.kt +++ b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/model/action/ReadWriteActionConfig.kt @@ -20,6 +20,7 @@ import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagemen import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.model.ManagedIndexMetaData import org.elasticsearch.client.Client import org.elasticsearch.cluster.service.ClusterService +import org.elasticsearch.common.settings.Settings import org.elasticsearch.common.xcontent.ToXContent import org.elasticsearch.common.xcontent.ToXContentObject import org.elasticsearch.common.xcontent.XContentBuilder @@ -47,6 +48,7 @@ data class ReadWriteActionConfig( clusterService: ClusterService, scriptService: ScriptService, client: Client, + settings: Settings, managedIndexMetaData: ManagedIndexMetaData ): Action = ReadWriteAction(clusterService, client, managedIndexMetaData, this) diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/model/action/ReplicaCountActionConfig.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/model/action/ReplicaCountActionConfig.kt index 6c3dd032d..426191788 100644 --- a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/model/action/ReplicaCountActionConfig.kt +++ b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/model/action/ReplicaCountActionConfig.kt @@ -22,6 +22,7 @@ import org.elasticsearch.client.Client import org.elasticsearch.cluster.service.ClusterService import org.elasticsearch.common.io.stream.StreamInput import org.elasticsearch.common.io.stream.StreamOutput +import org.elasticsearch.common.settings.Settings import org.elasticsearch.common.xcontent.ToXContent import org.elasticsearch.common.xcontent.ToXContentObject import org.elasticsearch.common.xcontent.XContentBuilder @@ -53,6 +54,7 @@ data class ReplicaCountActionConfig( clusterService: ClusterService, scriptService: ScriptService, client: Client, + settings: Settings, managedIndexMetaData: ManagedIndexMetaData ): Action = ReplicaCountAction(clusterService, client, managedIndexMetaData, this) diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/model/action/RolloverActionConfig.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/model/action/RolloverActionConfig.kt index 3a68febaf..b418c35f0 100644 --- a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/model/action/RolloverActionConfig.kt +++ b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/model/action/RolloverActionConfig.kt @@ -22,6 +22,7 @@ import org.elasticsearch.client.Client import org.elasticsearch.cluster.service.ClusterService import org.elasticsearch.common.io.stream.StreamInput import org.elasticsearch.common.io.stream.StreamOutput +import org.elasticsearch.common.settings.Settings import org.elasticsearch.common.unit.ByteSizeValue import org.elasticsearch.common.unit.TimeValue import org.elasticsearch.common.xcontent.ToXContent @@ -62,6 +63,7 @@ data class RolloverActionConfig( clusterService: ClusterService, scriptService: ScriptService, client: Client, + settings: Settings, managedIndexMetaData: ManagedIndexMetaData ): Action = RolloverAction(clusterService, client, managedIndexMetaData, this) diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/model/action/RollupActionConfig.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/model/action/RollupActionConfig.kt index 21ae3f26a..3438deadd 100644 --- a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/model/action/RollupActionConfig.kt +++ b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/model/action/RollupActionConfig.kt @@ -23,6 +23,7 @@ import org.elasticsearch.client.Client import org.elasticsearch.cluster.service.ClusterService import org.elasticsearch.common.io.stream.StreamInput import org.elasticsearch.common.io.stream.StreamOutput +import org.elasticsearch.common.settings.Settings import org.elasticsearch.common.xcontent.ToXContent import org.elasticsearch.common.xcontent.ToXContentObject import org.elasticsearch.common.xcontent.XContentBuilder @@ -50,6 +51,7 @@ class RollupActionConfig( clusterService: ClusterService, scriptService: ScriptService, client: Client, + settings: Settings, managedIndexMetaData: ManagedIndexMetaData ): Action = RollupAction(clusterService, client, managedIndexMetaData, this) diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/model/action/SnapshotActionConfig.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/model/action/SnapshotActionConfig.kt index 7b84f3d98..c8006b26e 100644 --- a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/model/action/SnapshotActionConfig.kt +++ b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/model/action/SnapshotActionConfig.kt @@ -22,6 +22,7 @@ import org.elasticsearch.client.Client import org.elasticsearch.cluster.service.ClusterService import org.elasticsearch.common.io.stream.StreamInput import org.elasticsearch.common.io.stream.StreamOutput +import org.elasticsearch.common.settings.Settings import org.elasticsearch.common.xcontent.ToXContent import org.elasticsearch.common.xcontent.ToXContentObject import org.elasticsearch.common.xcontent.XContentBuilder @@ -52,6 +53,7 @@ data class SnapshotActionConfig( clusterService: ClusterService, scriptService: ScriptService, client: Client, + settings: Settings, managedIndexMetaData: ManagedIndexMetaData ): Action = SnapshotAction(clusterService, client, managedIndexMetaData, this) diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/model/action/TransitionsActionConfig.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/model/action/TransitionsActionConfig.kt index cc2070a7a..923934556 100644 --- a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/model/action/TransitionsActionConfig.kt +++ b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/model/action/TransitionsActionConfig.kt @@ -21,6 +21,7 @@ import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagemen import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.model.Transition import org.elasticsearch.client.Client import org.elasticsearch.cluster.service.ClusterService +import org.elasticsearch.common.settings.Settings import org.elasticsearch.script.ScriptService data class TransitionsActionConfig( @@ -31,6 +32,7 @@ data class TransitionsActionConfig( clusterService: ClusterService, scriptService: ScriptService, client: Client, + settings: Settings, managedIndexMetaData: ManagedIndexMetaData ): Action = TransitionsAction(clusterService, client, managedIndexMetaData, this) } diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/model/destination/Destination.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/model/destination/Destination.kt index ba995e2bd..482734871 100644 --- a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/model/destination/Destination.kt +++ b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/model/destination/Destination.kt @@ -22,6 +22,7 @@ import com.amazon.opendistroforelasticsearch.alerting.destination.message.Custom import com.amazon.opendistroforelasticsearch.alerting.destination.message.SlackMessage import com.amazon.opendistroforelasticsearch.alerting.destination.response.DestinationResponse import com.amazon.opendistroforelasticsearch.indexmanagement.elasticapi.convertToMap +import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.util.isHostInDenylist import org.apache.logging.log4j.LogManager import org.elasticsearch.common.io.stream.StreamInput import org.elasticsearch.common.io.stream.StreamOutput @@ -115,7 +116,7 @@ data class Destination( } @Throws(IOException::class) - fun publish(compiledSubject: String?, compiledMessage: String): DestinationResponse { + fun publish(compiledSubject: String?, compiledMessage: String, denyHostRanges: List): DestinationResponse { val destinationMessage: BaseMessage when (type) { DestinationType.CHIME -> { @@ -144,6 +145,7 @@ data class Destination( .withMessage(compiledMessage).build() } } + validateDestinationUri(destinationMessage, denyHostRanges) val response = Notification.publish(destinationMessage) as DestinationResponse logger.info("Message published for action type: $type, messageid: ${response.responseContent}, statuscode: ${response.statusCode}") return response @@ -161,4 +163,10 @@ data class Destination( } return content } + + private fun validateDestinationUri(destinationMessage: BaseMessage, denyHostRanges: List) { + if (destinationMessage.isHostInDenylist(denyHostRanges)) { + throw IllegalArgumentException("The destination address is invalid.") + } + } } diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/settings/ManagedIndexSettings.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/settings/ManagedIndexSettings.kt index 7135c50c7..1253ae9d2 100644 --- a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/settings/ManagedIndexSettings.kt +++ b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/settings/ManagedIndexSettings.kt @@ -28,6 +28,7 @@ class ManagedIndexSettings { private val ALLOW_LIST_ALL = ActionConfig.ActionType.values().toList().map { it.type } val ALLOW_LIST_NONE = emptyList() val SNAPSHOT_DENY_LIST_NONE = emptyList() + const val HOST_DENY_LIST = "opendistro.destination.host.deny_list" val INDEX_STATE_MANAGEMENT_ENABLED: Setting = Setting.boolSetting( "opendistro.index_state_management.enabled", diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/step/notification/AttemptNotificationStep.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/step/notification/AttemptNotificationStep.kt index be6976965..d736969d4 100644 --- a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/step/notification/AttemptNotificationStep.kt +++ b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/step/notification/AttemptNotificationStep.kt @@ -19,12 +19,14 @@ import com.amazon.opendistroforelasticsearch.indexmanagement.elasticapi.convertT import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.model.ManagedIndexMetaData import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.model.action.NotificationActionConfig import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.model.managedindexmetadata.StepMetaData +import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.settings.ManagedIndexSettings import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.step.Step import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.withContext import org.apache.logging.log4j.LogManager import org.elasticsearch.client.Client import org.elasticsearch.cluster.service.ClusterService +import org.elasticsearch.common.settings.Settings import org.elasticsearch.script.Script import org.elasticsearch.script.ScriptService import org.elasticsearch.script.TemplateScript @@ -33,6 +35,7 @@ class AttemptNotificationStep( val clusterService: ClusterService, val scriptService: ScriptService, val client: Client, + val settings: Settings, val config: NotificationActionConfig, managedIndexMetaData: ManagedIndexMetaData ) : Step("attempt_notification", managedIndexMetaData) { @@ -40,6 +43,7 @@ class AttemptNotificationStep( private val logger = LogManager.getLogger(javaClass) private var stepStatus = StepStatus.STARTING private var info: Map? = null + private val hostDenyList = settings.getAsList(ManagedIndexSettings.HOST_DENY_LIST) override fun isIdempotent() = false @@ -47,7 +51,7 @@ class AttemptNotificationStep( override suspend fun execute(): AttemptNotificationStep { try { withContext(Dispatchers.IO) { - config.destination.publish(null, compileTemplate(config.messageTemplate, managedIndexMetaData)) + config.destination.publish(null, compileTemplate(config.messageTemplate, managedIndexMetaData), hostDenyList) } // publish internally throws an error for any invalid responses so its safe to assume if we reach this point it was successful diff --git a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/util/ManagedIndexUtils.kt b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/util/ManagedIndexUtils.kt index 6890b72e8..c3282faae 100644 --- a/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/util/ManagedIndexUtils.kt +++ b/src/main/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/util/ManagedIndexUtils.kt @@ -16,6 +16,7 @@ @file:JvmName("ManagedIndexUtils") package com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.util +import com.amazon.opendistroforelasticsearch.alerting.destination.message.BaseMessage import com.amazon.opendistroforelasticsearch.indexmanagement.IndexManagementPlugin.Companion.INDEX_MANAGEMENT_INDEX import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.ManagedIndexCoordinator import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.action.Action @@ -38,6 +39,8 @@ import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagemen import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.step.Step import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.step.delete.AttemptDeleteStep import com.amazon.opendistroforelasticsearch.jobscheduler.spi.schedule.IntervalSchedule +import inet.ipaddr.IPAddressString +import org.apache.logging.log4j.LogManager import org.elasticsearch.action.DocWriteRequest import org.elasticsearch.action.delete.DeleteRequest import org.elasticsearch.action.index.IndexRequest @@ -46,6 +49,7 @@ import org.elasticsearch.action.update.UpdateRequest import org.elasticsearch.client.Client import org.elasticsearch.cluster.metadata.IndexMetadata import org.elasticsearch.cluster.service.ClusterService +import org.elasticsearch.common.settings.Settings import org.elasticsearch.common.unit.ByteSizeValue import org.elasticsearch.common.unit.TimeValue import org.elasticsearch.common.xcontent.ToXContent @@ -54,6 +58,7 @@ import org.elasticsearch.index.query.BoolQueryBuilder import org.elasticsearch.index.query.QueryBuilders import org.elasticsearch.script.ScriptService import org.elasticsearch.search.builder.SearchSourceBuilder +import java.net.InetAddress import java.time.Instant import java.time.temporal.ChronoUnit @@ -233,6 +238,7 @@ fun State.getActionToExecute( clusterService: ClusterService, scriptService: ScriptService, client: Client, + settings: Settings, managedIndexMetaData: ManagedIndexMetaData ): Action? { var actionConfig: ActionConfig? @@ -254,14 +260,14 @@ fun State.getActionToExecute( // TODO: Refactor so we can get isLastStep from somewhere besides an instantiated Action class so we can simplify this to a when block // If stepCompleted is true and this is the last step of the action then we should get the next action if (managedIndexMetaData.stepMetaData != null && managedIndexMetaData.stepMetaData.stepStatus == Step.StepStatus.COMPLETED) { - val action = actionConfig.toAction(clusterService, scriptService, client, managedIndexMetaData) + val action = actionConfig.toAction(clusterService, scriptService, client, settings, managedIndexMetaData) if (action.isLastStep(managedIndexMetaData.stepMetaData.name)) { actionConfig = this.actions.getOrNull(managedIndexMetaData.actionMetaData.index + 1) ?: TransitionsActionConfig(this.transitions) } } } - return actionConfig.toAction(clusterService, scriptService, client, managedIndexMetaData) + return actionConfig.toAction(clusterService, scriptService, client, settings, managedIndexMetaData) } fun State.getUpdatedStateMetaData(managedIndexMetaData: ManagedIndexMetaData): StateMetaData { @@ -493,3 +499,17 @@ fun Policy.getDisallowedActions(allowList: List): List { * Allowed actions are ones that are specified in the [ManagedIndexSettings.ALLOW_LIST] setting. */ fun Action.isAllowed(allowList: List): Boolean = allowList.contains(this.type.type) + +private val baseMessageLogger = LogManager.getLogger(BaseMessage::class.java) + +fun BaseMessage.isHostInDenylist(networks: List): Boolean { + val ipStr = IPAddressString(this.uri.host) + for (network in networks) { + val netStr = IPAddressString(network) + if (netStr.contains(ipStr)) { + baseMessageLogger.error("Host: {} resolves to: {} which is in denylist: {}.", uri.host, InetAddress.getByName(uri.host), netStr) + return true + } + } + return false +} diff --git a/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/util/ManagedIndexUtilsTests.kt b/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/util/ManagedIndexUtilsTests.kt index 548fafbce..4fcf6c9df 100644 --- a/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/util/ManagedIndexUtilsTests.kt +++ b/src/test/kotlin/com/amazon/opendistroforelasticsearch/indexmanagement/indexstatemanagement/util/ManagedIndexUtilsTests.kt @@ -15,6 +15,8 @@ package com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.util +import com.amazon.opendistroforelasticsearch.alerting.destination.message.BaseMessage +import com.amazon.opendistroforelasticsearch.alerting.destination.message.CustomWebhookMessage import com.amazon.opendistroforelasticsearch.indexmanagement.IndexManagementPlugin.Companion.INDEX_MANAGEMENT_INDEX import com.amazon.opendistroforelasticsearch.indexmanagement.elasticapi.parseWithType import com.amazon.opendistroforelasticsearch.indexmanagement.indexstatemanagement.model.Conditions @@ -197,8 +199,59 @@ class ManagedIndexUtilsTests : ESTestCase() { .evaluateConditions(indexCreationDate = Instant.ofEpochMilli(-1L), numDocs = null, indexSize = null, transitionStartTime = Instant.now())) } + fun `test ips in denylist`() { + val ips = listOf( + "127.0.0.1", // 127.0.0.0/8 + "10.0.0.1", // 10.0.0.0/8 + "10.11.12.13", // 10.0.0.0/8 + "172.16.0.1", // "172.16.0.0/12" + "192.168.0.1", // 192.168.0.0/16" + "0.0.0.1", // 0.0.0.0/8 + "9.9.9.9" + ) + for (ip in ips) { + val bm = createMessageWithHost(ip) + assertEquals(true, bm.isHostInDenylist(HOST_DENY_LIST)) + } + } + + fun `test url in denylist`() { + val urls = listOf("https://www.amazon.com", "https://mytest.com", "https://mytest.com") + for (url in urls) { + val bm = createMessageWithURl(url) + assertEquals(false, bm.isHostInDenylist(HOST_DENY_LIST)) + } + } + private fun contentParser(bytesReference: BytesReference): XContentParser { return XContentHelper.createParser(xContentRegistry(), LoggingDeprecationHandler.INSTANCE, bytesReference, XContentType.JSON) } + + private val HOST_DENY_LIST = listOf( + "127.0.0.0/8", + "10.0.0.0/8", + "172.16.0.0/12", + "192.168.0.0/16", + "0.0.0.0/8", + "9.9.9.9" // ip + ) + + private fun createMessageWithHost(host: String): BaseMessage { + return CustomWebhookMessage.Builder("abc") + .withHost(host) + .withPath("incomingwebhooks/383c0e2b-d028-44f4-8d38-696754bc4574") + .withMessage("{\"Content\":\"Message test\"}") + .withMethod("POST") + .withQueryParams(HashMap()).build() + } + + private fun createMessageWithURl(url: String): BaseMessage { + return CustomWebhookMessage.Builder("abc") + .withUrl(url) + .withPath("incomingwebhooks/383c0e2b-d028-44f4-8d38-696754bc4574") + .withMessage("{\"Content\":\"Message test\"}") + .withMethod("POST") + .withQueryParams(HashMap()).build() + } }