diff --git a/conf/rate_limit_by_size.conf b/conf/rate_limit_by_size.conf new file mode 100644 index 00000000000..627f12c6caa --- /dev/null +++ b/conf/rate_limit_by_size.conf @@ -0,0 +1,26 @@ +[SERVICE] + Flush 1 + Daemon Off + Log_Level info + Parsers_File parsers.conf + +[INPUT] + Name tail + Path /var/log/containers/*.log + Parser docker + Tag kube.* + Mem_Buf_Limit 5MB + +[FILTER] + Name kubernetes + Match kube.* + +[FILTER] + Name lua + Match kube.* + script rate_limit.lua + call rate_limit_by_size + +[OUTPUT] + Name stdout + Match * diff --git a/scripts/rate_limit.lua b/scripts/rate_limit.lua index 1474d113d85..9f1f8d5085d 100644 --- a/scripts/rate_limit.lua +++ b/scripts/rate_limit.lua @@ -1,26 +1,32 @@ --[[ This Lua script is to do the rate limiting of logs based on some key. The Throttle filter in fluent-bit doesn't allow to do the rate limiting based on key - - sample configuration: - [FILTER] - Name lua - Match kube.* - script rate_limit.lua - call rate_limit ]] local counter = {} local time = 0 local group_key = "docker_id" -- Used to group logs. Groups are rate limited independently. +local rate_limit_field = "log" -- The field in the record whose size is used to determine the rate limit local group_bucket_period_s = 60 -- This is the period of of time in seconds over which group_bucket_limit applies. local group_bucket_limit = 6000 -- Maximum number logs allowed per groups over the period of group_bucket_period_s. - --- with above values, each and every containers running on the kubernetes will have a limit of 6000 logs for every 60 seconds since contianers have unique kubernetes.docker_id value +local group_bucket_limit_bytes = 30000 -- Maximum size of rate_limit_field in bytes allowed per kubernetes.group_key over the period of group_bucket_period_s. local function get_current_time(timestamp) return math.floor(timestamp / group_bucket_period_s) end +--[[ + This function is used to rate limit logs based on the number of logs of kubernetes.group_key. + If the number of logs in a group exceeds group_bucket_limit, the log is dropped. + E.g. With above values for the local variables, each and every containers running on Kubernetes will + have a limit of 6000 logs for every 60 seconds since contianers have unique kubernetes.docker_id value + + sample configuration: + [FILTER] + Name lua + Match kube.* + script rate_limit.lua + call rate_limit +]] function rate_limit(tag, timestamp, record) local t = os.time() local current_time = get_current_time(t) @@ -39,4 +45,40 @@ function rate_limit(tag, timestamp, record) end end return 0, 0, 0 -- keep the log -end \ No newline at end of file +end + +--[[ + This function is used to rate limit logs based on the size of the content of kubernetes.group_key. + E.g. With above values for the local variables, each and every container running on Kubernetes will + have a limit of 30000 bytes for every 60 seconds. + + sample configuration: + [FILTER] + Name lua + Match kube.* + script rate_limit.lua + call rate_limit_by_size +]] +function rate_limit_by_size(tag, timestamp, record) + local t = os.time() + local current_time = get_current_time(t) + if current_time ~= time then + time = current_time + counter = {} -- reset the counter + end + + if counter[group_key] == -1 then + return -1, 0, 0 -- Log group already rate limited. Hence drop it. + else + if counter[group_key] == nil then + counter[group_key] = #record[rate_limit_field] + else + counter[group_key] = counter[group_key] + #record[rate_limit_field] + end + if counter[group_key] > group_bucket_limit_bytes then + counter[group_key] = -1 -- value of -1 indicates that this group has been rate limited + return -1, 0, 0 -- drop the log + end + return 0, 0, 0 + end +end