elastic search + logstash + kibana

19/10/2019

elastic search

java -version
apt-get update
apt-get install default-jre
apt-get install default-jdk

wget -qO - https://artifacts.elastic.co/GPG-KEY-elasticsearch | apt-key add -
echo "deb https://artifacts.elastic.co/packages/6.x/apt stable main" | tee -a /etc/apt/sources.list.d/elastic-6.x.list
apt-get update
apt-get install elasticsearch

emacs /etc/elasticsearch/elasticsearch.yml
network.host: localhost

systemctl start elasticsearch
systemctl enable elasticsearch

check working

curl "localhost:9200"

get indexes

curl 'localhost:9200/_aliases?pretty=true'
curl 'localhost:9200/_cat/indices'

get types

curl -s -XGET 'http://localhost:9200/_mapping' | jq 'to_entries | .[] | {(.key): .value.mappings | keys}'

empty indexes

curl -XDELETE 'http://localhost:9200/filebeat-*'

empty pipelines

curl -XDELETE 'http://localhost:9200/_ingest/pipeline/*'

find something by id

curl 'http://localhost:9200/logstash-node-2019.05.16/doc/YiBswmoBrpq531GuVhD-?pretty'

search something

curl 'http://localhost:9200/logstash-node-*/doc/_search?pretty'
curl 'http://localhost:9200/logstash-node-*/_search?pretty'
curl 'http://localhost:9200/logstash-node-2019.05.16/_search?pretty'

insert

curl -H 'Content-Type: application/json' -XPOST 'http://localhost:9200/logstash-node-2019.05.16/doc?pretty' -d '{ "message" : "value", "@timestamp" : "2019-05-16T20:42:10.000Z"}'

kibana

apt-get install kibana
systemctl enable kibana
systemctl start kibana

Vérifier que ça marche

curl http://localhost:5601/status

Rendre accessible de l'exterieur

emacs /etc/apache2/sites-enabled/000-default.conf

    ServerName kibana.raphaelpiccolo.com

    ProxyRequests Off
    ProxyPass / http://localhost:5601/
    ProxyPassReverse / http://localhost:5601/
    ProxyPreserveHost On

    
        AuthName "Safe Zone"
        AuthType Basic
        AuthUserFile "/etc/apache2/.htpasswd"
        Require valid-user

logstash

apt-get install logstash

pour que le user logstash ait accès à /var/log/*

usermod -a -G adm logstash

Existing logformat options : https://httpd.apache.org/docs/2.4/fr/mod/mod_log_config.html
Existing grok patterns : https://github.com/elastic/logstash/blob/v1.4.2/patterns/grok-patterns

integrate apache access logs

emacs /etc/apache2/apache2.conf
LogFormat "%v:%p %h %l %u %t \"%r\" %>s %O \"%{Referer}i\" \"%{User-Agent}i\" %D" vhost_combined

tail -f /var/log/apache2/other_vhosts_access.log
fake.fr:80 176.159.8.38 - - [05/May/2019:16:24:48 +0200] "GET /test HTTP/1.1" 200 839 "-" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.131 Safari/537.36" 314

emacs /etc/logstash/conf.d/apache-access.conf
input {
  file {
    type => "apache-access"
    path => "/var/log/apache2/other_vhosts_access.log"
    start_position => beginning
  }
}

filter {
  if [type] == "apache-access" {
    grok {
      match => { "message" => "%{IPORHOST:vhost}:%{POSINT:port} %{IPORHOST:clientip} %{USER:ident} %{USER:auth} \[%{HTTPDATE:timestamp}\] \"(?:%{WORD:verb} %{NOTSPACE:request}(?: HTTP/%{NUMBER:httpversion})?|%{DATA:rawrequest})\" %{NUMBER:response} (?:%{NUMBER:bytes}|-) %{QS:referrer} %{QS:agent} %{NUMBER:duration}" }
    }
    date {
      match => [ "timestamp" , "dd/MMM/yyyy:HH:mm:ss Z" ]
    }
    mutate {
      remove_field => [ "timestamp" ]
    }

    if [agent]  {
      useragent {
        source => "agent"
        target => "useragent"
      }
    }

    if [clientip] and [clientip] != '::1' and [clientip] != '127.0.0.1' {
      geoip {
        source => "clientip"
      }
      mutate {
        convert => [ "responsetime", "integer" ]
        convert => [ "bytes", "integer" ]
      }
    }
  }
}

output {
  if [type] == "apache-access" {
    elasticsearch {
      hosts => ["127.0.0.1"]
      index => "logstash-apache-%{+YYYY.MM.dd}"
    }
  }
}

verifier la conf

sudo -u logstash /usr/share/logstash/bin/logstash --path.settings /etc/logstash -t -f /etc/logstash/conf.d/apache-access.conf
lancer en console pour tester
sudo -u logstash /usr/share/logstash/bin/logstash --path.settings /etc/logstash -f /etc/logstash/conf.d/apache-access.conf

activer en daemon

systemctl start logstash
systemctl enable logstash

verifier que l'index se remplit

curl 'localhost:9200/_cat/indices'

integrate apache error logs

tail -f /var/log/apache2/error.log
[Sun May 05 17:49:27.410788 2019] [proxy:error] [pid 486] [client 66.249.76.120:52362] AH00898: Error reading from remote server returned by /en/activity.json, referer: https://ideaz.world/en/activities/48

emacs /etc/logstash/conf.d/apache-error.conf
input {
  file {
    type => "apache-error"
    path => "/var/log/apache2/error.log"
    start_position => beginning
  }
}

filter {
  if [type] == "apache-error" {
    grok {
      match => { "message" => "%{HTTPD_ERRORLOG}" }
    }
    date {
      match => [ "timestamp", "EEE MMM dd HH:mm:ss.SSSSSS yyyy" ]
    }
    mutate {
      remove_field => [ "timestamp" ]
    }

    if [clientip] and [clientip] != '::1' and [clientip] != '127.0.0.1' {
      geoip {
        source => "clientip"
      }
    }
  }
}

output {
  if [type] == "apache-error" {
    elasticsearch {
      hosts => ["127.0.0.1"]
      index => "logstash-apache-%{+YYYY.MM.dd}"
    }
  }
}

recevoir du json sur un port en tcp

emacs /etc/logstash/conf.d/apache-error.conf
input {
  tcp {
    type => "node"
    port => 12345
    codec => json
  }
}

filter {
  if [type] == "node" {
    date {
      match => ["date", "yyyy-MM-dd'T'HH:mm:ss'.'SSSZ" ]
    }

    if [clientip] and [clientip] != '::1' and [clientip] != '127.0.0.1' {
      geoip {
        source => "clientip"
      }
    }
  }
}

output {
  stdout { codec => rubydebug }
}

output {
  elasticsearch {
    hosts => ["127.0.0.1"]
    index => "logstash-node-%{+YYYY.MM.dd}"
  }
}

debug en console

emacs test.conf
input { stdin { } }
output { stdout { codec => rubydebug } }
filter {
  grok {
    match => { "message" => "%{NUMBER:numero}" }
  }
}
output {
  stdout { codec => rubydebug }
}
echo "3456789" | sudo -u logstash /usr/share/logstash/bin/logstash --path.settings /etc/logstash -f /etc/logstash/conf.d/test.conf
{
          "host" => "ideaz.world",
        "numero" => "3456789",
      "@version" => "1",
       "message" => "[3456789]",
    "@timestamp" => 2019-05-05T20:39:33.799Z
}

filebeat

apt-get install filebeat
emacs /etc/filebeat/filebeat.yml
setup.kibana:
    host: "localhost:5601"

filebeat modules enable apache2
filebeat modules list
filebeat setup
filebeat setup -e
systemctl start filebeat
systemctl enable filebeat

pour activer autre chose :

filebeat modules enable system
filebeat modules enable redis
filebeat modules enable mysql
filebeat modules enable mongodb
filebeat setup
systemctl restart filebeat

metricbeat

apt-get install metricbeat
emacs /etc/metricbeat/metricbeat.yml
setup.kibana:
    host: "localhost:5601"

metricbeat modules enable system
metricbeat modules enable apache
metricbeat modules enable mongodb
metricbeat modules enable mysql
metricbeat modules enable redis

metricbeat setup

systemctl restart metricbeat

Raccourcis

elastic search
kibana
logstash
filebeat
metricbeat