[Elastic][observability] Filebeat

설명

• https://www.elastic.co/guide/en/beats/filebeat/current/index.html
• 경량 로그 수집기

yaml

    ---
    apiVersion: v1
    kind: ConfigMap
    metadata:
      name: filebeat-config
      namespace: default
      labels:
        k8s-app: filebeat
    data:
      filebeat.yml: |-
        filebeat.inputs:
        - type: container
          paths:
            - /var/log/containers/elasticsearch*.log
          processors:
            - add_kubernetes_metadata:
                host: ${NODE_NAME}
                matchers:
                - logs_path:
                    logs_path: "/var/log/containers/"
          multiline.pattern: '\{"type":'
          multiline.negate: true
          multiline.match: after
        - type: container
          paths:
            - /var/log/containers/metricbeat*.log
          processors:
            - add_kubernetes_metadata:
                host: ${NODE_NAME}
                matchers:
                - logs_path:
                    logs_path: "/var/log/containers/"
        processors:
          - add_cloud_metadata:
          - add_host_metadata:
        setup.ilm:
          enabled: true
          rollover_alias: "filebeat"
          pattern: "{now/d}-000001"
          check_exists: true
          overwrite: true
          #overwrite: false
          policy_name: "filebeat-ilm-policy"
          policy_file: /usr/share/filebeat/ilm-policy.json
        setup.template.enabled: true
        setup.template.json.enabled: true
        setup.template.json.path: "/usr/share/filebeat/template.json"
        setup.template.json.name: "filebeat-template"
        setup.template.overwrite: true
        output.elasticsearch:
          hosts: ["http://elasticsearch:9200"]
        #output.logstash:
          #enabled: true
          #hosts: ["logstash:5044"]
    
    ---
    apiVersion: v1
    kind: ConfigMap
    metadata:
      name: filebeat-ilm-policy
      namespace: default
    data:
      ilm-policy.json: |
        {
        "policy": {
            "phases": {
            "hot": {
                "actions": {
                "rollover": {
                    "max_size": "5gb",
                    "max_age": "1d"
                }
                }
            },
            "delete": {
                "min_age": "365d",
                "actions": {
                "delete": {}
                }
            }
            }
        }
        }
    
    ---
    apiVersion: v1
    kind: ConfigMap
    metadata:
      name: filebeat-template
      namespace: default
    data:
      template.json: |
        {
        "index_patterns": [
          "filebeat-*"
        ],
        "settings" : {
          "index" : {
            "number_of_shards" : "1",
            "number_of_replicas" : "1",
            "lifecycle.name": "filebeat-ilm-policy",
            "lifecycle.rollover_alias": "filebeat"
          }
        }
        }
    
    ---
    apiVersion: apps/v1
    kind: DaemonSet
    metadata:
      name: filebeat
      namespace: default
      labels:
        k8s-app: filebeat
    spec:
      selector:
        matchLabels:
          k8s-app: filebeat
      template:
        metadata:
          labels:
            k8s-app: filebeat
        spec:
          serviceAccountName: filebeat
          terminationGracePeriodSeconds: 30
          hostNetwork: true
          dnsPolicy: ClusterFirstWithHostNet
          tolerations:
            - key: node-role.kubernetes.io/master
              operator: Exists
              effect: NoSchedule
            - key: node.kubernetes.io/not-ready
              operator: Exists
              effect: NoSchedule
          containers:
          - name: filebeat
            image: elastic/filebeat:7.14.0
            args: [
              "-c", "/etc/filebeat.yml",
              "-e",
            ]
            env:
            - name: ELASTICSEARCH_HOST
              value: http://elasticsearch
            - name: ELASTICSEARCH_PORT
              value: "9200"
            - name: ELASTICSEARCH_USERNAME
              value:
            - name: ELASTICSEARCH_PASSWORD
              value:
            - name: ELASTIC_CLOUD_ID
              value:
            - name: ELASTIC_CLOUD_AUTH
              value:
            - name: NODE_NAME
              valueFrom:
                fieldRef:
                  fieldPath: spec.nodeName
            securityContext:
              runAsUser: 0
            resources:
              limits:
                memory: 200Mi
              requests:
                cpu: 100m
                memory: 100Mi
            volumeMounts:
            - name: config
              mountPath: /etc/filebeat.yml
              readOnly: true
              subPath: filebeat.yml
            - name: data
              mountPath: /usr/share/filebeat/data
            - name: varlibdockercontainers
              mountPath: /var/lib/docker/containers
              readOnly: true
            - name: varlog
              mountPath: /var/log
              readOnly: true
            - name: template
              mountPath: /usr/share/filebeat/template.json
              subPath: template.json
            - name: ilm-policy
              mountPath: /usr/share/filebeat/ilm-policy.json
              subPath: ilm-policy.json
          volumes:
          - name: config
            configMap:
              defaultMode: 0640
              name: filebeat-config
          - name: varlibdockercontainers
            hostPath:
              path: /var/lib/docker/containers
          - name: varlog
            hostPath:
              path: /var/log
          - name: data
            hostPath:
              path: /var/lib/filebeat-data
              type: DirectoryOrCreate
          - name: template
            configMap:
              name: filebeat-template
          - name: ilm-policy
            configMap:
              name: filebeat-ilm-policy
    
    ---
    apiVersion: rbac.authorization.k8s.io/v1
    kind: ClusterRoleBinding
    metadata:
      name: filebeat
    subjects:
    - kind: ServiceAccount
      name: filebeat
      namespace: default
    roleRef:
      kind: ClusterRole
      name: filebeat
      apiGroup: rbac.authorization.k8s.io
    
    ---
    apiVersion: rbac.authorization.k8s.io/v1
    kind: RoleBinding
    metadata:
      name: filebeat
      namespace: default
    subjects:
      - kind: ServiceAccount
        name: filebeat
        namespace: default
    roleRef:
      kind: Role
      name: filebeat
      apiGroup: rbac.authorization.k8s.io
    
    ---
    apiVersion: rbac.authorization.k8s.io/v1
    kind: RoleBinding
    metadata:
      name: filebeat-kubeadm-config
      namespace: default
    subjects:
      - kind: ServiceAccount
        name: filebeat
        namespace: default
    roleRef:
      kind: Role
      name: filebeat-kubeadm-config
      apiGroup: rbac.authorization.k8s.io
    
    ---
    apiVersion: rbac.authorization.k8s.io/v1
    kind: ClusterRole
    metadata:
      name: filebeat
      labels:
        k8s-app: filebeat
    rules:
    - apiGroups: [""]
      resources:
      - namespaces
      - pods
      - nodes
      verbs:
      - get
      - watch
      - list
    - apiGroups: ["apps"]
      resources:
        - replicasets
      verbs: ["get", "list", "watch"]
    
    ---
    apiVersion: rbac.authorization.k8s.io/v1
    kind: Role
    metadata:
      name: filebeat
      namespace: default
      labels:
        k8s-app: filebeat
    rules:
      - apiGroups:
          - coordination.k8s.io
        resources:
          - leases
        verbs: ["get", "create", "update"]
    
    ---
    apiVersion: rbac.authorization.k8s.io/v1
    kind: Role
    metadata:
      name: filebeat-kubeadm-config
      namespace: default
      labels:
        k8s-app: filebeat
    rules:
      - apiGroups: [""]
        resources:
          - configmaps
        resourceNames:
          - kubeadm-config
        verbs: ["get"]
    
    ---
    apiVersion: v1
    kind: ServiceAccount
    metadata:
      name: filebeat
      namespace: default
      labels:
        k8s-app: filebeat