[Elastic][observability] Logstash
Updated:
설명
- 실시간 파이프라이닝 기능을 갖춘 오픈 소스 데이터 수집 엔진
- 다양한 소스에서 데이터 수집 및 필터 후 다양한 저장소로 출력
- 입력
- https://www.elastic.co/guide/en/logstash/current/input-plugins.html
- 형식이나 복잡성과 관계 없이 데이터 수집 가능
- beats, elasticsearch, exec, file, github, heartbeat, http, imap, jdbc, kafka, log4j, syslog, …
- 필터
- https://www.elastic.co/guide/en/logstash/current/filter-plugins.html
- grok
- 비정형 데이터 분석 후 정형 데이터를 생성
- geoip
- IP 주소에 대한 지리적 정보를 추가
- …
- 출력
- https://www.elastic.co/guide/en/logstash/current/output-plugins.html
- 데이터를 특정 대상으로 전달
- 파이프라인의 마지막 단계
- csv, elasticsearch, email, exec, kafka, stdout, syslog, …
- Persistent Queue
- https://www.elastic.co/guide/en/logstash/current/persistent-queues.html
- 정상/비정상 종료 시 메세지 손실에 대해 최소 1회 전달을 보장
- DLQ
- https://www.elastic.co/guide/en/logstash/current/dead-letter-queues.html
- 처리할 수 없는 이벤트 발생 시 데이터 유실을 방지하기 위해 삭제 대신 DLQ에 저장 가능
- DLQ에 저장된 이벤트 내용으로는 원본 이벤트, 미처리 이유, 큐에 저장된 시간 등이 포함
- 현재 Elasticsearch 출력만 지원
yaml
- filebeat와 metricbeat의 데이터를 받아서 각각 다른 alias로 elasticsearch에 저장
- filebeat는 에러 로그 여부를 판단하여 http 전송
- metricbeat는 정적 매핑을 위해 템플릿 설정
apiVersion: v1 kind: ConfigMap metadata: name: logstash-configmap namespace: default data: logstash.yml: | log.level: debug http.host: "127.0.0.1" #path.config: /usr/share/logstash/pipeline pipeline.workers: 2 pipelines.yml: | - pipeline.id: main path.config: "/usr/share/logstash/pipeline/main.yml" - pipeline.id: filebeat path.config: "/usr/share/logstash/pipeline/filebeat.yml" - pipeline.id: filebeat_http path.config: "/usr/share/logstash/pipeline/filebeat-http.yml" - pipeline.id: metricbeat path.config: "/usr/share/logstash/pipeline/metricbeat.yml" main.yml: | input { beats { port => 5044 } } output { if [agent][type] == "filebeat" { pipeline { send_to => filebeat } } else if [agent][type] == "metricbeat" { pipeline { send_to => metricbeat } } } filebeat.yml: | input { pipeline { address => filebeat } } output { if "level=error" in [message] or "[ERROR]" in [message] or [message] =~ /^E\d* *([01][0-9]|2[0-3]):([0-5][0-9]):([0-5][0-9])/ { pipeline { send_to => filebeat_http } } elasticsearch { hosts => ["http://elasticsearch:9200"] ilm_rollover_alias => "filebeat" ilm_pattern => "{now/d}-000001" ilm_policy => "filebeat-ilm-policy" } } filebeat-http.yml: | input { pipeline { address => filebeat_http } } output { http { url => "http://127.0.0.1:10000/test" content_type => "application/json" http_method => "post" format => "json" #mapping => ["field1", "value1", "text", "%{message}"] #headers => ["Authorization", "xxx"] message => "%{message}" pool_max => 1000 pool_max_per_route => 500 } } metricbeat.yml: | input { pipeline { address => metricbeat } } output { elasticsearch { hosts => ["http://elasticsearch:9200"] ilm_rollover_alias => "metricbeat-prometheus" ilm_pattern => "{now/d}-000001" ilm_policy => "metricbeat-prometheus-ilm-policy" template => "/usr/share/logstash/template/template.json" template_name => "metricbeat-prometheus-template" template_overwrite => true } } --- apiVersion: v1 kind: ConfigMap metadata: name: metricbeat-prometheus-template namespace: default data: template.json: | { "index_patterns": [ "metricbeat-prometheus-*" ], "settings" : { "index" : { "number_of_shards" : "1", "number_of_replicas" : "1", "lifecycle.name": "metricbeat-prometheus-ilm-policy", "lifecycle.rollover_alias": "metricbeat-prometheus" } }, "mappings" : { "properties" : { "@timestamp" : { "type" : "date" }, "agent" : { "properties" : { "ephemeral_id" : { "type" : "text", "fields" : { "keyword" : { "type" : "keyword", "ignore_above" : 256 } } }, "hostname" : { "type" : "text", "fields" : { "keyword" : { "type" : "keyword", "ignore_above" : 256 } } }, "id" : { "type" : "text", "fields" : { "keyword" : { "type" : "keyword", "ignore_above" : 256 } } }, "name" : { "type" : "text", "fields" : { "keyword" : { "type" : "keyword", "ignore_above" : 256 } } }, "type" : { "type" : "text", "fields" : { "keyword" : { "type" : "keyword", "ignore_above" : 256 } } }, "version" : { "type" : "text", "fields" : { "keyword" : { "type" : "keyword", "ignore_above" : 256 } } } } }, "ecs" : { "properties" : { "version" : { "type" : "text", "fields" : { "keyword" : { "type" : "keyword", "ignore_above" : 256 } } } } }, "event" : { "properties" : { "dataset" : { "type" : "text", "fields" : { "keyword" : { "type" : "keyword", "ignore_above" : 256 } } }, "duration" : { "type" : "long" }, "module" : { "type" : "text", "fields" : { "keyword" : { "type" : "keyword", "ignore_above" : 256 } } } } }, "host" : { "properties" : { "name" : { "type" : "text", "fields" : { "keyword" : { "type" : "keyword", "ignore_above" : 256 } } } } }, "metricset" : { "properties" : { "name" : { "type" : "text", "fields" : { "keyword" : { "type" : "keyword", "ignore_above" : 256 } } }, "period" : { "type" : "long" } } }, "prometheus" : { "properties" : { "labels" : { "properties" : { "area" : { "type" : "text", "fields" : { "keyword" : { "type" : "keyword", "ignore_above" : 256 } } }, "branch" : { "type" : "text", "fields" : { "keyword" : { "type" : "keyword", "ignore_above" : 256 } } }, "breaker" : { "type" : "text", "fields" : { "keyword" : { "type" : "keyword", "ignore_above" : 256 } } }, "build_date" : { "type" : "date" }, "build_hash" : { "type" : "text", "fields" : { "keyword" : { "type" : "keyword", "ignore_above" : 256 } } }, "cache" : { "type" : "text", "fields" : { "keyword" : { "type" : "keyword", "ignore_above" : 256 } } }, "cluster" : { "type" : "text", "fields" : { "keyword" : { "type" : "keyword", "ignore_above" : 256 } } }, "cluster_uuid" : { "type" : "text", "fields" : { "keyword" : { "type" : "keyword", "ignore_above" : 256 } } }, "code" : { "type" : "text", "fields" : { "keyword" : { "type" : "keyword", "ignore_above" : 256 } } }, "color" : { "type" : "text", "fields" : { "keyword" : { "type" : "keyword", "ignore_above" : 256 } } }, "device" : { "type" : "text", "fields" : { "keyword" : { "type" : "keyword", "ignore_above" : 256 } } }, "es_client_node" : { "type" : "text", "fields" : { "keyword" : { "type" : "keyword", "ignore_above" : 256 } } }, "es_data_node" : { "type" : "text", "fields" : { "keyword" : { "type" : "keyword", "ignore_above" : 256 } } }, "es_ingest_node" : { "type" : "text", "fields" : { "keyword" : { "type" : "keyword", "ignore_above" : 256 } } }, "es_master_node" : { "type" : "text", "fields" : { "keyword" : { "type" : "keyword", "ignore_above" : 256 } } }, "gc" : { "type" : "text", "fields" : { "keyword" : { "type" : "keyword", "ignore_above" : 256 } } }, "goversion" : { "type" : "text", "fields" : { "keyword" : { "type" : "keyword", "ignore_above" : 256 } } }, "handler" : { "type" : "text", "fields" : { "keyword" : { "type" : "keyword", "ignore_above" : 256 } } }, "host" : { "type" : "text", "fields" : { "keyword" : { "type" : "keyword", "ignore_above" : 256 } } }, "index" : { "type" : "text", "fields" : { "keyword" : { "type" : "keyword", "ignore_above" : 256 } } }, "instance" : { "type" : "text", "fields" : { "keyword" : { "type" : "keyword", "ignore_above" : 256 } } }, "job" : { "type" : "text", "fields" : { "keyword" : { "type" : "keyword", "ignore_above" : 256 } } }, "lucene_version" : { "type" : "text", "fields" : { "keyword" : { "type" : "keyword", "ignore_above" : 256 } } }, "method" : { "type" : "text", "fields" : { "keyword" : { "type" : "keyword", "ignore_above" : 256 } } }, "mount" : { "type" : "text", "fields" : { "keyword" : { "type" : "keyword", "ignore_above" : 256 } } }, "name" : { "type" : "text", "fields" : { "keyword" : { "type" : "keyword", "ignore_above" : 256 } } }, "node" : { "type" : "text", "fields" : { "keyword" : { "type" : "keyword", "ignore_above" : 256 } } }, "path" : { "type" : "text", "fields" : { "keyword" : { "type" : "keyword", "ignore_above" : 256 } } }, "pool" : { "type" : "text", "fields" : { "keyword" : { "type" : "keyword", "ignore_above" : 256 } } }, "quantile" : { "type" : "text", "fields" : { "keyword" : { "type" : "keyword", "ignore_above" : 256 } } }, "revision" : { "type" : "text", "fields" : { "keyword" : { "type" : "keyword", "ignore_above" : 256 } } }, "role" : { "type" : "text", "fields" : { "keyword" : { "type" : "keyword", "ignore_above" : 256 } } }, "shard" : { "type" : "text", "fields" : { "keyword" : { "type" : "keyword", "ignore_above" : 256 } } }, "type" : { "type" : "text", "fields" : { "keyword" : { "type" : "keyword", "ignore_above" : 256 } } }, "url" : { "type" : "text", "fields" : { "keyword" : { "type" : "keyword", "ignore_above" : 256 } } }, "version" : { "type" : "text", "fields" : { "keyword" : { "type" : "keyword", "ignore_above" : 256 } } } } } } }, "service" : { "properties" : { "address" : { "type" : "text", "fields" : { "keyword" : { "type" : "keyword", "ignore_above" : 256 } } }, "type" : { "type" : "text", "fields" : { "keyword" : { "type" : "keyword", "ignore_above" : 256 } } } } } } } } --- apiVersion: apps/v1 kind: Deployment metadata: name: logstash namespace: default spec: replicas: 1 selector: matchLabels: app: logstash template: metadata: labels: app: logstash spec: containers: - name: logstash image: logstash:7.14.0 ports: - containerPort: 5044 volumeMounts: - name: config-volume mountPath: /usr/share/logstash/config/logstash.yml subPath: logstash.yml - name: logstash-pipeline-volume mountPath: /usr/share/logstash/config/pipelines.yml subPath: pipelines.yml - name: main-pipeline-volume mountPath: /usr/share/logstash/pipeline/main.yml subPath: main.yml - name: filebeat-pipeline-volume mountPath: /usr/share/logstash/pipeline/filebeat.yml subPath: filebeat.yml - name: filebeat-http-pipeline-volume mountPath: /usr/share/logstash/pipeline/filebeat-http.yml subPath: filebeat-http.yml - name: metricbeat-pipeline-volume mountPath: /usr/share/logstash/pipeline/metricbeat.yml subPath: metricbeat.yml - name: template-volume mountPath: /usr/share/logstash/template/template.json subPath: template.json volumes: - name: config-volume configMap: name: logstash-configmap items: - key: logstash.yml path: logstash.yml - name: logstash-pipeline-volume configMap: name: logstash-configmap items: - key: pipelines.yml path: pipelines.yml - name: main-pipeline-volume configMap: name: logstash-configmap items: - key: main.yml path: main.yml - name: filebeat-pipeline-volume configMap: name: logstash-configmap items: - key: filebeat.yml path: filebeat.yml - name: filebeat-http-pipeline-volume configMap: name: logstash-configmap items: - key: filebeat-http.yml path: filebeat-http.yml - name: metricbeat-pipeline-volume configMap: name: logstash-configmap items: - key: metricbeat.yml path: metricbeat.yml - name: template-volume configMap: name: metricbeat-prometheus-template --- apiVersion: v1 kind: Service metadata: name: logstash namespace: default spec: selector: app: logstash ports: - protocol: TCP port: 5044 targetPort: 5044 type: ClusterIP