fluentd -> elasticsearch -> kibana でログ解析

dockerを使いますよ！

HOSTNAME       IP             DOCKER-ID
HOST           192.168.0.1/24 ------------
elasticsearch  192.168.0.2/24 1e0000000000
kibana+nginx   192.168.0.3/24 2k0000000000 < --- http://192.168.0.3/
fluentd        192.168.0.4/24 3f0000000000

■piplineによるネットワーク
・とりあえずOSを起動しておく

sudo docker run -it -p 9200:9200 -it --name elasticsearch ubuntu:latest /bin/bash
Ctrl+p + Ctrl+q
sudo docker run -it -p 5601:5601 --name kibana ubuntu:latest /bin/bash
Ctrl+p + Ctrl+q
sudo docker run -it -p 80:80 --name fluentd ubuntu:latest /bin/bash
Ctrl+p + Ctrl+q

・dockerのidを確認しておく

sudo docker ps

・piplineを使ってNICを作成

ip addr add 192.168.0.1/24 dev br1
git clone https://github.com/jpetazzo/pipework.git
sudo ./pipework/pipework br1 1e0000000000 192.168.0.2/24
sudo ./pipework/pipework br1 2k0000000000 192.168.0.3/24
sudo ./pipework/pipework br1 3f0000000000 192.168.0.4/24

■elasticsearch

sudo docker attach 1e0000000000
--- in elasticsearch
apt-get update
apt-get install wget -y
apt-get install openjdk-7-jdk
wget https://download.elasticsearch.org/elasticsearch/elasticsearch/elasticsearch-1.3.4.deb
dpkg -i elasticsearch-1.3.4.deb
service elasticsearch start
---
Ctrl+p + Ctrl+q
sudo docker ps
sudo docker commit 1e0000000000 elasticsearch

■Kibana

sudo docker attach 2k0000000000
sudo apt-get install nginx -y
vi /etc/nginx/site-enable/default
---
server {
             listen *:80 ;
             server_name localhost;

             access_log /var/log/nginx/kibana.access.log;

             location /kibana/ {
                       root /usr/share/nginx/html;
                       index index.html index.htm;
             }

             location / {
             proxy_pass http://192.168.0.2:9200;
             proxy_read_timeout 90;
             }
 }
---
wget http://download.elasticsearch.org/kibana/kibana/kibana-latest.tar.gz
tar zxpf kibana-latest.tar.gz
ln -s kibana-latest /usr/share/nginx/html/kibana
Ctrl+p + Ctrl+q
sudo docker ps
sudo docker commit 2k0000000000 kibana

■fluentd

sudo docker attach 3f0000000000
sudo apt-get install nginx    < --- ログを出す為だけにいれるのだｗ
sudo apt-get install curl ruby-curb -y
sudo apt-get install ruby1.9.3
sudo gem install fluentd
sudo apt-get install libcurl4-gnutls-dev -y
sudo gem install fluent-plugin-elasticsearch
vi /etc/fluentd.conf
---
# syslog

type tail
path /var/log/syslog
pos_file /var/log/syslog.pos
tag syslog
format syslog


type elasticsearch
host 192.168.0.2
port 9200
type_name syslog
logstash_format true
logstash_prefix syslog
logstash_dateformat %Y%m
buffer_type memory
buffer_chunk_limit 10m
buffer_queue_limit 10
flush_interval 1s
retry_limit 16
retry_wait 1s


# nginx

type tail
path /var/log/nginx/access.log
pos_file /var/log/td-agent/httpd-access.log.pos
tag nginx.access
format nginx


type elasticsearch
host 192.168.0.2
port 9200
type_name nginx
logstash_format true
logstash_prefix nginx_access
logstash_dateformat %Y%m
buffer_type memory
buffer_chunk_limit 10m
buffer_queue_limit 10
flush_interval 1s
retry_limit 16
retry_wait 1s

---
sudo fluentd -c /etc/fluentd.conf
Ctrl+p + Ctrl+q
sudo docker ps
sudo docker commit 3f0000000000 kibana

■ちょこっとメモ
elasticsearchに入っているデータを覗く方法

curl -XGET http://192.168.0.2:9200/_search '
{
"query" : {
"match_all" : {}
}
}'