Upgrade aws component version

aws/aws-alb-ingress-controller/base/cluster-role.yaml

@@ -6,7 +6,6 @@ rules:
   - apiGroups:
       - ""
       - extensions
-      - networking.k8s.io
     resources:
       - configmaps
       - endpoints
@@ -33,4 +32,4 @@ rules:
     verbs:
       - get
       - list
-      - watch
+      - watch

aws/aws-alb-ingress-controller/base/deployment.yaml

@@ -48,6 +48,6 @@ spec:
             # List of regions: http://docs.aws.amazon.com/general/latest/gr/rande.html#vpc_region
             # - --aws-region=us-west-1
           # Repository location of the ALB Ingress Controller.
-          image: docker.io/amazon/aws-alb-ingress-controller:v1.1.2
+          image: docker.io/amazon/aws-alb-ingress-controller
           imagePullPolicy: Always
       serviceAccountName: alb-ingress-controller

aws/aws-alb-ingress-controller/base/kustomization.yaml

@@ -13,7 +13,7 @@ generatorOptions:
 images:
 - name: docker.io/amazon/aws-alb-ingress-controller
   newName: docker.io/amazon/aws-alb-ingress-controller
-  newTag: v1.1.2
+  newTag: v1.1.5
 configMapGenerator:
 - name: alb-ingress-controller-parameters
   env: params.env

aws/fluentd-cloud-watch/base/configmap.yaml

@@ -8,6 +8,8 @@ data:
   fluent.conf: |
     @include containers.conf
     @include systemd.conf
+    @include host.conf
+
     <match fluent.**>
       @type null
     </match>
@@ -17,6 +19,7 @@ data:
       @id in_tail_container_logs
       @label @containers
       path /var/log/containers/*.log
+      exclude_path ["/var/log/containers/cloudwatch-agent*", "/var/log/containers/fluentd*"]
       pos_file /var/log/fluentd-containers.log.pos
       tag *
       read_from_head true
@@ -25,23 +28,119 @@ data:
         time_format %Y-%m-%dT%H:%M:%S.%NZ
       </parse>
     </source>
+
+    <source>
+      @type tail
+      @id in_tail_cwagent_logs
+      @label @cwagentlogs
+      path /var/log/containers/cloudwatch-agent*
+      pos_file /var/log/cloudwatch-agent.log.pos
+      tag *
+      read_from_head true
+      <parse>
+        @type json
+        time_format %Y-%m-%dT%H:%M:%S.%NZ
+      </parse>
+    </source>
+
+    <source>
+      @type tail
+      @id in_tail_fluentd_logs
+      @label @fluentdlogs
+      path /var/log/containers/fluentd*
+      pos_file /var/log/fluentd.log.pos
+      tag *
+      read_from_head true
+      <parse>
+        @type json
+        time_format %Y-%m-%dT%H:%M:%S.%NZ
+      </parse>
+    </source>
+
+    <label @fluentdlogs>
+      <filter **>
+        @type kubernetes_metadata
+        @id filter_kube_metadata_fluentd
+      </filter>
+
+      <filter **>
+        @type record_transformer
+        @id filter_fluentd_stream_transformer
+        <record>
+          stream_name ${tag_parts[3]}
+        </record>
+      </filter>
+
+      <match **>
+        @type relabel
+        @label @NORMAL
+      </match>
+    </label>
+
     <label @containers>
       <filter **>
         @type kubernetes_metadata
         @id filter_kube_metadata
       </filter>
+
       <filter **>
         @type record_transformer
         @id filter_containers_stream_transformer
         <record>
           stream_name ${tag_parts[3]}
         </record>
       </filter>
+
+      <filter **>
+        @type concat
+        key log
+        multiline_start_regexp /^\S/
+        separator ""
+        flush_interval 5
+        timeout_label @NORMAL
+      </filter>
+
+      <match **>
+        @type relabel
+        @label @NORMAL
+      </match>
+    </label>
+
+    <label @cwagentlogs>
+      <filter **>
+        @type kubernetes_metadata
+        @id filter_kube_metadata_cwagent
+      </filter>
+
+      <filter **>
+        @type record_transformer
+        @id filter_cwagent_stream_transformer
+        <record>
+          stream_name ${tag_parts[3]}
+        </record>
+      </filter>
+
+      <filter **>
+        @type concat
+        key log
+        multiline_start_regexp /^\d{4}[-/]\d{1,2}[-/]\d{1,2}/
+        separator ""
+        flush_interval 5
+        timeout_label @NORMAL
+      </filter>
+
+      <match **>
+        @type relabel
+        @label @NORMAL
+      </match>
+    </label>
+
+    <label @NORMAL>
       <match **>
         @type cloudwatch_logs
         @id out_cloudwatch_logs_containers
         region "#{ENV.fetch('REGION')}"
-        log_group_name "/eks/#{ENV.fetch('CLUSTER_NAME')}/containers"
+        log_group_name "/aws/containerinsights/#{ENV.fetch('CLUSTER_NAME')}/application"
         log_stream_name_key stream_name
         remove_log_stream_name_key true
         auto_create_stream true
@@ -63,11 +162,16 @@ data:
         field_map {"MESSAGE": "message", "_HOSTNAME": "hostname", "_SYSTEMD_UNIT": "systemd_unit"}
         field_map_strict true
       </entry>
-      path /run/log/journal
-      pos_file /var/log/fluentd-journald-kubelet.pos
+      path /var/log/journal
+      <storage>
+        @type local
+        persistent true
+        path /var/log/fluentd-journald-kubelet-pos.json
+      </storage>
       read_from_head true
       tag kubelet.service
     </source>
+
     <source>
       @type systemd
       @id in_systemd_kubeproxy
@@ -77,11 +181,16 @@ data:
         field_map {"MESSAGE": "message", "_HOSTNAME": "hostname", "_SYSTEMD_UNIT": "systemd_unit"}
         field_map_strict true
       </entry>
-      path /run/log/journal
-      pos_file /var/log/fluentd-journald-kubeproxy.pos
+      path /var/log/journal
+      <storage>
+        @type local
+        persistent true
+        path /var/log/fluentd-journald-kubeproxy-pos.json
+      </storage>
       read_from_head true
       tag kubeproxy.service
     </source>
+
     <source>
       @type systemd
       @id in_systemd_docker
@@ -91,24 +200,35 @@ data:
         field_map {"MESSAGE": "message", "_HOSTNAME": "hostname", "_SYSTEMD_UNIT": "systemd_unit"}
         field_map_strict true
       </entry>
-      path /run/log/journal
-      pos_file /var/log/fluentd-journald-docker.pos
+      path /var/log/journal
+      <storage>
+        @type local
+        persistent true
+        path /var/log/fluentd-journald-docker-pos.json
+      </storage>
       read_from_head true
       tag docker.service
     </source>
+
     <label @systemd>
+      <filter **>
+        @type kubernetes_metadata
+        @id filter_kube_metadata_systemd
+      </filter>
+
       <filter **>
         @type record_transformer
         @id filter_systemd_stream_transformer
         <record>
           stream_name ${tag}-${record["hostname"]}
         </record>
       </filter>
+
       <match **>
         @type cloudwatch_logs
         @id out_cloudwatch_logs_systemd
         region "#{ENV.fetch('REGION')}"
-        log_group_name "/eks/#{ENV.fetch('CLUSTER_NAME')}/systemd"
+        log_group_name "/aws/containerinsights/#{ENV.fetch('CLUSTER_NAME')}/dataplane"
         log_stream_name_key stream_name
         auto_create_stream true
         remove_log_stream_name_key true
@@ -119,4 +239,74 @@ data:
           retry_forever true
         </buffer>
       </match>
+    </label>
+  host.conf: |
+    <source>
+      @type tail
+      @id in_tail_dmesg
+      @label @hostlogs
+      path /var/log/dmesg
+      pos_file /var/log/dmesg.log.pos
+      tag host.dmesg
+      read_from_head true
+      <parse>
+        @type syslog
+      </parse>
+    </source>
+
+    <source>
+      @type tail
+      @id in_tail_secure
+      @label @hostlogs
+      path /var/log/secure
+      pos_file /var/log/secure.log.pos
+      tag host.secure
+      read_from_head true
+      <parse>
+        @type syslog
+      </parse>
+    </source>
+
+    <source>
+      @type tail
+      @id in_tail_messages
+      @label @hostlogs
+      path /var/log/messages
+      pos_file /var/log/messages.log.pos
+      tag host.messages
+      read_from_head true
+      <parse>
+        @type syslog
+      </parse>
+    </source>
+
+    <label @hostlogs>
+      <filter **>
+        @type kubernetes_metadata
+        @id filter_kube_metadata_host
+      </filter>
+
+      <filter **>
+        @type record_transformer
+        @id filter_containers_stream_transformer_host
+        <record>
+          stream_name ${tag}-${record["host"]}
+        </record>
+      </filter>
+
+      <match host.**>
+        @type cloudwatch_logs
+        @id out_cloudwatch_logs_host_logs
+        region "#{ENV.fetch('REGION')}"
+        log_group_name "/aws/containerinsights/#{ENV.fetch('CLUSTER_NAME')}/host"
+        log_stream_name_key stream_name
+        remove_log_stream_name_key true
+        auto_create_stream true
+        <buffer>
+          flush_interval 5
+          chunk_limit_size 2m
+          queued_chunks_limit_size 32
+          retry_forever true
+        </buffer>
+      </match>
     </label>

aws/fluentd-cloud-watch/base/daemonset.yaml

@@ -24,17 +24,22 @@ spec:
               mountPath: /config-volume
             - name: fluentdconf
               mountPath: /fluentd/etc
+        - name: update-log-driver
+          image: busybox
+          command: ['sh','-c','']
       containers:
         - name: fluentd-cloudwatch
-          image: fluent/fluentd-kubernetes-daemonset:v1.1-debian-cloudwatch
+          image: fluent/fluentd-kubernetes-daemonset
           env:
             - name: REGION
               value: $(REGION)
             - name: CLUSTER_NAME
               value: $(CLUSTER_NAME)
+            - name: CI_VERSION
+              value: "k8s/1.0.1"
           resources:
             limits:
-              memory: 200Mi
+              memory: 400Mi
             requests:
               cpu: 100m
               memory: 200Mi
@@ -51,6 +56,9 @@ spec:
             - name: runlogjournal
               mountPath: /run/log/journal
               readOnly: true
+            - name: dmesg
+              mountPath: /var/log/dmesg
+              readOnly: true
       volumes:
         - name: config-volume
           configMap:
@@ -66,3 +74,6 @@ spec:
         - name: runlogjournal
           hostPath:
             path: /run/log/journal
+        - name: dmesg
+          hostPath:
+            path: /var/log/dmesg

aws/fluentd-cloud-watch/base/kustomization.yaml

@@ -14,7 +14,7 @@ generatorOptions:
 images:
 - name: fluent/fluentd-kubernetes-daemonset
   newName: fluent/fluentd-kubernetes-daemonset
-  newTag: v1.1-debian-cloudwatch
+  newTag: v1.7.3-debian-cloudwatch-1.0
 configMapGenerator:
 - name: fluentd-cloud-watch-parameters
   env: params.env

aws/infra_configs/cluster_config.yaml

@@ -7,7 +7,7 @@ metadata:
   # AWS_CLUSTER_NAME and AWS_REGION will override `name` and `region` here.
   name: your_cluster_name
   region: your_cluster_region
-  version: '1.13'
+  version: '1.14'
 # If your region has multiple availability zones, you can specify 3 of them.
 # cluster AZs must be set explicitly for single AZ nodegroup example to work
 #availabilityZones: ["us-west-2b", "us-west-2c", "us-west-2d"]

aws/nvidia-device-plugin/base/daemonset.yaml

@@ -22,7 +22,7 @@ spec:
           operator: Exists
           effect: NoSchedule
       containers:
-        - image: nvidia/k8s-device-plugin:1.0.0-beta
+        - image: nvidia/k8s-device-plugin:1.0.0-beta4
           name: nvidia-device-plugin-ctr
           securityContext:
             allowPrivilegeEscalation: false

aws/nvidia-device-plugin/base/kustomization.yaml

@@ -8,4 +8,4 @@ commonLabels:
 images:
 - name: nvidia/k8s-device-plugin
   newName: nvidia/k8s-device-plugin
-  newTag: 1.0.0-beta
+  newTag: 1.0.0-beta4