I am setting up fluentd and elasticsearch on a local VM in order to try the fluentd and ES stack.
OS: centos (recent)
[root@localhost data]# cat /etc/redhat-release
CentOS release 6.5 (Final)
I am elasticsearch up and running on localhost (I used it with logstash with no issue)
[root@localhost data]# curl -X GET http://localhost:9200/
{
"status" : 200,
"name" : "Simon Williams",
"version" : {
"number" : "1.2.1",
"build_hash" : "6c95b759f9e7ef0f8e17f77d850da43ce8a4b364",
"build_timestamp" : "2014-06-03T15:02:52Z",
"build_snapshot" : false,
"lucene_version" : "4.8"
},
"tagline" : "You Know, for Search"
}
I have installed td-agent following the installation notes from fluentd website. I am using that configuration file:
<source>
type tail
path /tmp/data/log
pos_file /tmp/data/log.pos
format /^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[\
^\"]*)" "(?<agent>[^\"]*)")?/
time_format %d/%b/%Y:%H:%M:%S %z
tag front.nginx.access
</source>
<match front.nginx.access>
type elasticsearch
host localhost
port 9200
index_name fluentd
type_name nginx
include_tag_key
# buffering
buffer_type file
buffer_path /tmp/fluentd/buffer/
flush_interval 10s
buffer_chunk_limit 16m
buffer_queue_limit 4096
retry_wait 15s
</match>
Here is the start-up log:
2014-07-24 13:39:58 +0200 [info]: starting fluentd-0.10.50
2014-07-24 13:39:58 +0200 [info]: reading config file path="/etc/td-agent/td-agent.conf"
2014-07-24 13:39:58 +0200 [info]: gem 'fluent-mixin-config-placeholders' version '0.2.4'
2014-07-24 13:39:58 +0200 [info]: gem 'fluent-mixin-plaintextformatter' version '0.2.6'
2014-07-24 13:39:58 +0200 [info]: gem 'fluent-plugin-elasticsearch' version '0.3.1'
2014-07-24 13:39:58 +0200 [info]: gem 'fluent-plugin-flume' version '0.1.1'
2014-07-24 13:39:58 +0200 [info]: gem 'fluent-plugin-mongo' version '0.7.3'
2014-07-24 13:39:58 +0200 [info]: gem 'fluent-plugin-parser' version '0.3.4'
2014-07-24 13:39:58 +0200 [info]: gem 'fluent-plugin-rewrite-tag-filter' version '1.4.1'
2014-07-24 13:39:58 +0200 [info]: gem 'fluent-plugin-s3' version '0.4.0'
2014-07-24 13:39:58 +0200 [info]: gem 'fluent-plugin-scribe' version '0.10.10'
2014-07-24 13:39:58 +0200 [info]: gem 'fluent-plugin-td' version '0.10.20'
2014-07-24 13:39:58 +0200 [info]: gem 'fluent-plugin-td-monitoring' version '0.1.2'
2014-07-24 13:39:58 +0200 [info]: gem 'fluent-plugin-webhdfs' version '0.2.2'
2014-07-24 13:39:58 +0200 [info]: gem 'fluentd' version '0.10.50'
2014-07-24 13:39:58 +0200 [info]: using configuration file: <ROOT>
<source>
type tail
path /tmp/data/log
pos_file /tmp/data/log.pos
format /^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?/
time_format %d/%b/%Y:%H:%M:%S %z
tag front.nginx.access
</source>
<match front.nginx.access>
type elasticsearch
host localhost
port 9200
index_name fluentd
type_name nginx
include_tag_key
buffer_type file
buffer_path /tmp/fluentd/buffer/
flush_interval 10s
buffer_chunk_limit 16m
buffer_queue_limit 4096
retry_wait 15s
</match>
</ROOT>
2014-07-24 13:39:58 +0200 [info]: adding source type="tail"
2014-07-24 13:39:58 +0200 [info]: adding match pattern="front.nginx.access" type="elasticsearch"
2014-07-24 13:39:58 +0200 [info]: following tail of /tmp/data/log
I get that error:
2014-07-24 13:40:00 +0200 [warn]: temporarily failed to flush the buffer. next_retry=2014-07-24 13:40:13 +0200 error_class="Elasticsearch::Transport::Transport::Errors::ServiceUnavailable" error="[503] " instance=70247139359260
2014-07-24 13:40:00 +0200 [warn]: /usr/lib64/fluent/ruby/lib/ruby/gems/1.9.1/gems/elasticsearch-transport-0.4.11/lib/elasticsearch/transport/transport/base.rb:132:in `__raise_transport_error'
2014-07-24 13:40:00 +0200 [warn]: /usr/lib64/fluent/ruby/lib/ruby/gems/1.9.1/gems/elasticsearch-transport-0.4.11/lib/elasticsearch/transport/transport/base.rb:227:in `perform_request'
2014-07-24 13:40:00 +0200 [warn]: /usr/lib64/fluent/ruby/lib/ruby/gems/1.9.1/gems/elasticsearch-transport-0.4.11/lib/elasticsearch/transport/transport/http/faraday.rb:20:in `perform_request'
2014-07-24 13:40:00 +0200 [warn]: /usr/lib64/fluent/ruby/lib/ruby/gems/1.9.1/gems/elasticsearch-transport-0.4.11/lib/elasticsearch/transport/client.rb:92:in `perform_request'
2014-07-24 13:40:00 +0200 [warn]: /usr/lib64/fluent/ruby/lib/ruby/gems/1.9.1/gems/elasticsearch-api-0.4.11/lib/elasticsearch/api/actions/ping.rb:19:in `ping'
2014-07-24 13:40:00 +0200 [warn]: /usr/lib64/fluent/ruby/lib/ruby/gems/1.9.1/gems/fluent-plugin-elasticsearch-0.3.1/lib/fluent/plugin/out_elasticsearch.rb:46:in `client'
2014-07-24 13:40:00 +0200 [warn]: /usr/lib64/fluent/ruby/lib/ruby/gems/1.9.1/gems/fluent-plugin-elasticsearch-0.3.1/lib/fluent/plugin/out_elasticsearch.rb:103:in `send'
2014-07-24 13:40:00 +0200 [warn]: /usr/lib64/fluent/ruby/lib/ruby/gems/1.9.1/gems/fluent-plugin-elasticsearch-0.3.1/lib/fluent/plugin/out_elasticsearch.rb:98:in `write'
2014-07-24 13:40:00 +0200 [warn]: /usr/lib64/fluent/ruby/lib/ruby/gems/1.9.1/gems/fluentd-0.10.50/lib/fluent/buffer.rb:296:in `write_chunk'
2014-07-24 13:40:00 +0200 [warn]: /usr/lib64/fluent/ruby/lib/ruby/gems/1.9.1/gems/fluentd-0.10.50/lib/fluent/buffer.rb:276:in `pop'
2014-07-24 13:40:00 +0200 [warn]: /usr/lib64/fluent/ruby/lib/ruby/gems/1.9.1/gems/fluentd-0.10.50/lib/fluent/output.rb:310:in `try_flush'
2014-07-24 13:40:00 +0200 [warn]: /usr/lib64/fluent/ruby/lib/ruby/gems/1.9.1/gems/fluentd-0.10.50/lib/fluent/output.rb:132:in `run'
running tcpdump on port 9200, I get nothing...
tcpdump -x -X -i any 'port 9200'
setenforce Permissive
and see if it helps. – Alcanzar