logstash grok sample

 

logstash pipeline.conf sample

• kv filter(key value filter) 사용.
• grok 패턴 대신 dissect 사용하여 성능 향상.

input {
        beats {
                port => 5044
        }
}



filter {
	if [fb_type] not in ["aa", "bb"] {
			drop { }
	}
	
	if [fb_type] == "aa" {
		grok {
			match => { "message" => "^\[(?<fb_time>\d{4}-%{MONTHNUM}-%{MONTHDAY} \d{2}:\d{2}:\d{2}\.\d{3})\] \[%{USERNAME:fb_name}\] .*" }
		}

		if [fb_name] == "T->P" {

			kv {
				field_split => "&?"
				source => "message"
				trim_value => "\[\]"
			}


			if [KEY1] {
				dissect {
					mapping => {"KEY1" => "%{}_%{msg1}_%{msg2}"}
				}
				mutate {
					add_field => {"msg" => ["%{[msg1]}", "%{[msg3]}"]}
					convert => {"msg" => "string"}
					remove_field => ["msg1", "msg2"]
				}
			}

			if [KEY2] {
				grok {
					match => { "KEY2" => "^(?<data1>\d+}) (?<data2>\w{1,8})" }
				}
			}
			if [KEY3] {
				dissect {
					mapping => {"KEY3" => "%{data1}_%{data2}"}
				}
			}
		}

		date {
			match => ["fb_time", "YYYY-MM-dd HH:mm:ss.SSS"]
			target => "@timestamp"
			timezone => 'Asia/Seoul'
		}

	}

	if [fb_type] == "bb" {
		grok {
			match => {
				"message" =>
					"^\[(?<bb_time>\d{4}-%{MONTHNUM}-%{MONTHDAY} \d{2}:\d{2}:\d{2}\.\d{3})\d+\] \[%{LOGLEVEL:log_level}\] \((?<source>.*)\) \[?(?<msg>[a-zA-Z0-9_]+)\]?.*"
			}
		}
		
		date {
			match => ["bb_time", "YYYY-MM-dd HH:mm:ss.SSS"]
			target => "@timestamp"
			timezone => 'Asia/Seoul'
		}
	}

}

output {

        if [fb_type] == "aa" {
                elasticsearch {
                        hosts => "elasticsearch:9200"
                        index => "fb_aa-%{+YYYY.MM}"
                        user => "user"
                        password => "pass"
                }
        } else if [fb_type] == "bb" {
                elasticsearch {
                        hosts => "elasticsearch:9200"
                        index => "fb_bb-%{+YYYY.MM}"
                        user => "user"
                        password => "pass"
                }
        }

}

 

 

filebeat filebeat.yml

filebeat 에서 아래와 같이 fileds 에 사용자 정의 field 를 추가하여

logstash 의 grok 패턴을 줄여 cpu 를 덜 쓸 수 있음.

filebeat.inputs:
- type: log
  enabled: true
  paths:
    - /var/app/app1.log
    - /var/app/app2.log

  fields_under_root: true
  fields:
    fb_type: aa

filebeat.config.modules:
  path: ${path.config}/modules.d/*.yml
  reload.enabled: false

setup.template.settings:
  index.number_of_shards: 1

output.logstash:
  hosts: ["logstash_ip_or_domain:5044"]

processors:
  - add_host_metadata: ~
  - add_cloud_metadata: ~