jenksli / learnscrapy Goto Github PK

input {
kafka {
bootstrap_servers => "[kafka IP]:9092"
topics => ["Your Topic"]
consumer_threads => 2
group_id => "Your group"
}
}

filter {
json {
source => "message"
}
kv {
source => "data"
field_split => "&"
}
urldecode {
all_fields => true
}
translate {
field => "opeType"
destination => "opeType_mes"
refresh_interval => 60
dictionary_path => "[config path]"
override => true
}

    translate {
            field => "status"
            destination => "status_mes"
            refresh_interval => 60
            dictionary_path => "[config path]"
            override => true
    }
    mutate {
            remove_field => ["data", "message"]
            }

}
output {
elasticsearch {
hosts => ["[Elasticsearch IP]:9200"]
index => "app1-logs-%{+YYYY.MM.dd}"
}
}

input {
kafka {
bootstrap_servers => "[kafka IP]:9092"
topics => ["Your Topic"]
consumer_threads => 2
group_id => "Your group"
}
}

filter {
json {
source => "message"
}
kv {
source => "data"
field_split => "&"
}
urldecode {
all_fields => true
}
translate {
field => "opeType"
destination => "opeType_mes"
refresh_interval => 60
dictionary_path => "[config path]"
override => true
}

    translate {
            field => "status"
            destination => "status_mes"
            refresh_interval => 60
            dictionary_path => "[config path]"
            override => true
    }
    mutate {
            remove_field => ["data", "message"]
            }

}
output {
elasticsearch {
hosts => ["[Elasticsearch IP]:9200"]
index => "app1-logs-%{+YYYY.MM.dd}"
}
}