Logstash doesn't load csv data to elastic search

1
votes

I am not able to load data to an elastic search node on localhost with logstash... I want to have logstash reading csv files and loading those data to elastic search. But nothing works, I can only read data that I add to elastic search manually, it seems that logstash do nothing.

My logstash config is this:

input {
    file {
        path => [ "C:\Users\Michele\Downloads\logstash-1.5.3\logstash-1.5.3\Users\*.csv" ]
        
        start_position => "beginning"
    }
}

filter {
    csv {
        columns => ["timestamp", "impianto", "tipo_misura", "valore","unita_misura"]
        separator => ","
    }
}

output {
    elasticsearch {
        action => "index"
        host => "localhost"
        cluster => "elasticsearch"
        node_name => "NCC-1701-A"
        index => "myindex"
        index_type => "pompe"
        workers => 1
    }
}

My csv file is:

2015-08-03T18:46:00,Abbiategrasso,Pressione gruppo 1,55.5,m
2015-08-03T18:46:10,Abbiategrasso,Pressione gruppo 1,44.4,m
2015-08-03T18:46:20,Abbiategrasso,Pressione gruppo 1,66.6,m
2015-08-03T18:46:30,Abbiategrasso,Pressione gruppo 1,33.3,m
2015-08-03T18:46:40,Abbiategrasso,Pressione gruppo 1,22.2,m
2015-08-03T18:46:50,Abbiategrasso,Pressione gruppo 1,77.7,m
2015-08-03T18:47:00,Abbiategrasso,Pressione gruppo 1,11.1,m
2015-08-03T18:47:10,Abbiategrasso,Pressione gruppo 1,44.4,m
2015-08-03T18:47:20,Abbiategrasso,Pressione gruppo 1,55.5,m
2015-08-03T18:47:30,Abbiategrasso,Pressione gruppo 1,33.3,m
2015-08-03T18:47:40,Abbiategrasso,Pressione gruppo 1,22.2,m
2015-08-03T18:47:50,Abbiategrasso,Pressione gruppo 1,66.6,m
2015-08-03T18:48:00,Abbiategrasso,Pressione gruppo 1,33.3,m
2015-08-03T18:48:10,Abbiategrasso,Pressione gruppo 1,77.7,m
2015-08-03T18:48:20,Abbiategrasso,Pressione gruppo 1,22.2,m
2015-08-03T18:48:30,Abbiategrasso,Pressione gruppo 1,88.8,m
2015-08-03T18:48:40,Abbiategrasso,Pressione gruppo 1,55.5,m
2015-08-03T18:48:50,Abbiategrasso,Pressione gruppo 1,33.3,m
2015-08-03T18:49:00,Abbiategrasso,Pressione gruppo 1,55.5,m

Nothing new appear on "myindex" index! But I don't know why...

EDIT

Changed logstash config file from "*.csv" that doesn't get files to "abbiategrasso.csv"

My --debug message on logstash try now, is:

←[33mfailed action with response of 400, dropping action: ["index", {:_id=>nil,
:_index=>"abbiategrasso", :_type=>"pompe", :_routing=>nil}, #<LogStash::Event:0x
1cea7b7 @metadata_accessors=#<LogStash::Util::Accessors:0x1e577ee @store={"path"
=>"C:\\Users\\Michele\\Downloads\\logstash-1.5.3\\logstash-1.5.3\\Users\\abbiate
grasso.csv", "retry_count"=>0}, @lut={"[path]"=>[{"path"=>"C:\\Users\\Michele\\D
ownloads\\logstash-1.5.3\\logstash-1.5.3\\Users\\abbiategrasso.csv", "retry_coun
t"=>0}, "path"]}>, @cancelled=false, @data={"message"=>["2015-08-03T18:48:00,Abb
iategrasso,Pressione gruppo 1,66.6,m\r"], "@version"=>"1", "@timestamp"=>"2015-0
9-07T17:28:32.501Z", "host"=>"Michele-HP", "path"=>"C:\\Users\\Michele\\Download
s\\logstash-1.5.3\\logstash-1.5.3\\Users\\abbiategrasso.csv", "timestamp"=>"2015
-08-03T18:48:00", "impianto"=>"Abbiategrasso", "tipo_misura"=>"Pressione gruppo
1", "valore"=>"66.6", "unita_misura"=>"m"}, @metadata={"path"=>"C:\\Users\\Miche
le\\Downloads\\logstash-1.5.3\\logstash-1.5.3\\Users\\abbiategrasso.csv", "retry
_count"=>0}, @accessors=#<LogStash::Util::Accessors:0x2ff785 @store={"message"=>
["2015-08-03T18:48:00,Abbiategrasso,Pressione gruppo 1,66.6,m\r"], "@version"=>"
1", "@timestamp"=>"2015-09-07T17:28:32.501Z", "host"=>"Michele-HP", "path"=>"C:\
\Users\\Michele\\Downloads\\logstash-1.5.3\\logstash-1.5.3\\Users\\abbiategrasso
.csv", "timestamp"=>"2015-08-03T18:48:00", "impianto"=>"Abbiategrasso", "tipo_mi
sura"=>"Pressione gruppo 1", "valore"=>"66.6", "unita_misura"=>"m"}, @lut={"host
"=>[{"message"=>["2015-08-03T18:48:00,Abbiategrasso,Pressione gruppo 1,66.6,m\r"
], "@version"=>"1", "@timestamp"=>"2015-09-07T17:28:32.501Z", "host"=>"Michele-H
P", "path"=>"C:\\Users\\Michele\\Downloads\\logstash-1.5.3\\logstash-1.5.3\\User
s\\abbiategrasso.csv", "timestamp"=>"2015-08-03T18:48:00", "impianto"=>"Abbiateg
rasso", "tipo_misura"=>"Pressione gruppo 1", "valore"=>"66.6", "unita_misura"=>"
m"}, "host"], "path"=>[{"message"=>["2015-08-03T18:48:00,Abbiategrasso,Pressione
 gruppo 1,66.6,m\r"], "@version"=>"1", "@timestamp"=>"2015-09-07T17:28:32.501Z",
 "host"=>"Michele-HP", "path"=>"C:\\Users\\Michele\\Downloads\\logstash-1.5.3\\l
ogstash-1.5.3\\Users\\abbiategrasso.csv", "timestamp"=>"2015-08-03T18:48:00", "i
mpianto"=>"Abbiategrasso", "tipo_misura"=>"Pressione gruppo 1", "valore"=>"66.6"
, "unita_misura"=>"m"}, "path"], "message"=>[{"message"=>["2015-08-03T18:48:00,A
bbiategrasso,Pressione gruppo 1,66.6,m\r"], "@version"=>"1", "@timestamp"=>"2015
-09-07T17:28:32.501Z", "host"=>"Michele-HP", "path"=>"C:\\Users\\Michele\\Downlo
ads\\logstash-1.5.3\\logstash-1.5.3\\Users\\abbiategrasso.csv", "timestamp"=>"20
15-08-03T18:48:00", "impianto"=>"Abbiategrasso", "tipo_misura"=>"Pressione grupp
o 1", "valore"=>"66.6", "unita_misura"=>"m"}, "message"], "timestamp"=>[{"messag
e"=>["2015-08-03T18:48:00,Abbiategrasso,Pressione gruppo 1,66.6,m\r"], "@version
"=>"1", "@timestamp"=>"2015-09-07T17:28:32.501Z", "host"=>"Michele-HP", "path"=>
"C:\\Users\\Michele\\Downloads\\logstash-1.5.3\\logstash-1.5.3\\Users\\abbiategr
asso.csv", "timestamp"=>"2015-08-03T18:48:00", "impianto"=>"Abbiategrasso", "tip
o_misura"=>"Pressione gruppo 1", "valore"=>"66.6", "unita_misura"=>"m"}, "timest
amp"], "impianto"=>[{"message"=>["2015-08-03T18:48:00,Abbiategrasso,Pressione gr
uppo 1,66.6,m\r"], "@version"=>"1", "@timestamp"=>"2015-09-07T17:28:32.501Z", "h
ost"=>"Michele-HP", "path"=>"C:\\Users\\Michele\\Downloads\\logstash-1.5.3\\logs
tash-1.5.3\\Users\\abbiategrasso.csv", "timestamp"=>"2015-08-03T18:48:00", "impi
anto"=>"Abbiategrasso", "tipo_misura"=>"Pressione gruppo 1", "valore"=>"66.6", "
unita_misura"=>"m"}, "impianto"], "tipo_misura"=>[{"message"=>["2015-08-03T18:48
:00,Abbiategrasso,Pressione gruppo 1,66.6,m\r"], "@version"=>"1", "@timestamp"=>
"2015-09-07T17:28:32.501Z", "host"=>"Michele-HP", "path"=>"C:\\Users\\Michele\\D
ownloads\\logstash-1.5.3\\logstash-1.5.3\\Users\\abbiategrasso.csv", "timestamp"
=>"2015-08-03T18:48:00", "impianto"=>"Abbiategrasso", "tipo_misura"=>"Pressione
gruppo 1", "valore"=>"66.6", "unita_misura"=>"m"}, "tipo_misura"], "valore"=>[{"
message"=>["2015-08-03T18:48:00,Abbiategrasso,Pressione gruppo 1,66.6,m\r"], "@v
ersion"=>"1", "@timestamp"=>"2015-09-07T17:28:32.501Z", "host"=>"Michele-HP", "p
ath"=>"C:\\Users\\Michele\\Downloads\\logstash-1.5.3\\logstash-1.5.3\\Users\\abb
iategrasso.csv", "timestamp"=>"2015-08-03T18:48:00", "impianto"=>"Abbiategrasso"
, "tipo_misura"=>"Pressione gruppo 1", "valore"=>"66.6", "unita_misura"=>"m"}, "
valore"], "unita_misura"=>[{"message"=>["2015-08-03T18:48:00,Abbiategrasso,Press
ione gruppo 1,66.6,m\r"], "@version"=>"1", "@timestamp"=>"2015-09-07T17:28:32.50
1Z", "host"=>"Michele-HP", "path"=>"C:\\Users\\Michele\\Downloads\\logstash-1.5.
3\\logstash-1.5.3\\Users\\abbiategrasso.csv", "timestamp"=>"2015-08-03T18:48:00"
, "impianto"=>"Abbiategrasso", "tipo_misura"=>"Pressione gruppo 1", "valore"=>"6
6.6", "unita_misura"=>"m"}, "unita_misura"]}>>] {:level=>:warn, :file=>"/Users/M
ichele/Downloads/logstash-1.5.3/logstash-1.5.3/vendor/bundle/jruby/1.9/gems/logs
tash-output-elasticsearch-1.0.5-java/lib/logstash/outputs/elasticsearch.rb", :li
ne=>"531", :method=>"submit"}←[0m

Logstash fail to upload data...

ACHIEVED

Remove current data from your index if you have charged data manually with Marvel.Sense for example... for this use a DELETE command of your index. Next time logstash will be able to create a new index that meets it's requirements!

Also for file definition in logstash config file don't use "*.csv" it doesn't work... it doesnt search all files in the folder, it search for the named file like that :(

My config now is:

input {
    file {
        path => [ "C:\Users\Michele\Downloads\logstash-1.5.3\logstash-1.5.3\Users\abbiategrasso4.csv" ]
        
        start_position => "beginning"
    }
}

filter {
    csv {
        columns => ["timestamp", "impianto", "tipo_misura", "valore","unita_misura"]
        separator => ","
    }
    
    mutate {
        convert => { "valore" => "float" }
    }
}

output {
    elasticsearch {
        action => "index"
        host => "localhost"
        cluster => "elasticsearch"
        node_name => "NCC-1701-A"
        index => "abbiategrasso"
        document_type => "pompe"
        workers => 1
    }
    
    stdout { codec => rubydebug }
}

Now works, I'll try Kibana :)

2
csvelasticsearchlogstash

2 Answers

1
votes

While you are troubleshooting I would start by printing to standard out.

output { stdout { codec => rubydebug } }

This way you can see if the issue is with logstash or with your interface to elasticsearch.

I've also found it highly useful to play around with the sincedb when doing file inputs. It's possible that while you were trouble shooting you read the data in and the sincedb was modified. Add the following line to your file {} while troubleshooting. Though don't forget to remove it when you go into actual usage or you will re-ingest data that you don't intend to.

sincedb_path => "/dev/null"
0
votes

The configuration for your elasticsearch output plugin seems to have an issue, i.e. index_type should be document_type instead:

output {
    elasticsearch {
        action => "index"
        host => "localhost"
        cluster => "elasticsearch"
        node_name => "NCC-1701-A"
        index => "myindex"
        document_type => "pompe"        <----- fix this
        workers => 1
    }
}

Also in addition to what bravosierra99 said, you should start your logstash with --debug so you get some TRACE level information on what's happening under the hood. If the above fix still doesn't work, I'd suggest you update your question with the debug log output you get from logstash.