geo_point in Elastic

Question

I'm trying to map a latitude and longitude to a geo_point in Elastic.

Here's my log file entry:

13-01-2017 ORDER COMPLETE: £22.00 Glasgow, 55.856299, -4.258845

And here's my conf file

input {
file {
  path => "/opt/logs/orders.log"
  start_position => "beginning"
 }
}

filter {
   grok {
       match => { "message" => "(?<date>[0-9-]+) (?<order_status>ORDER [a-zA-Z]+): (?<order_amount>£[0-9.]+) (?<order_location>[a-zA-Z ]+)"}
}

mutate {
       convert => { "order_amount" => "float" }
       convert => { "order_lat" => "float" }
       convert => { "order_long" => "float" }

       rename => {
                  "order_long" => "[location][lon]"
                  "order_lat" => "[location][lat]"
       }
 }
}

output {
      elasticsearch {
               hosts => "localhost"

               index => "sales"
               document_type => "order"

     }
    stdout {}
}

I start logstash with /bin/logstash -f orders.conf and this gives:

"@version"=>{"type"=>"keyword", "include_in_all"=>false}, "geoip"=>{"dynamic"=>true,
"properties"=>{"ip"=>{"type"=>"ip"},
"location"=>{"type"=>"geo_point"}, "latitude"=>{"type"=>"half_float"},
"longitude"=>{"type"=>"half_float"}}}}}}}}

See? It's seeing location as a geo_point. Yet GET sales/_mapping results in this:

"location": {
        "properties": {
          "lat": {
            "type": "text",
            "fields": {
              "keyword": {
                "type": "keyword",
                "ignore_above": 256
              }
            }
          },
          "lon": {
            "type": "text",
            "fields": {
              "keyword": {
                "type": "keyword",
                "ignore_above": 256
              }
            }
          }
        }
      },

Update

Each time I reindex, I stop logstash thenremove the .sincedb from /opt/logstash/data/plugins/inputs/file.... I have also made a brand new log file and I increment the index each time (I'm currently up to sales7).

conf file

input {
   file {
     path => "/opt/ag-created/logs/orders2.log"
     start_position => "beginning"
   }
}

filter {
  grok {
     match => { "message" => "(?<date>[0-9-]+) (?<order_status>ORDER [a-zA-Z]+): (?<order_amount>£[0-9.]+) (?<order_location>[a-zA-Z ]+), (?<order_lat>[0-9.]+), (?<order_long>[-0-9.]+)( - (?<order_failure_reason>[A-Za-z :]+))?" }
}

 mutate {
    convert => { "order_amount" => "float" }
 }

 mutate {
    convert => { "order_lat" => "float" }
 }

 mutate {
    convert => { "order_long" => "float" }
 }

 mutate {
     rename => { "order_long" => "[location][lon]" }
 }

 mutate {
     rename => { "order_lat" => "[location][lat]" }
  }
 }

output {
      elasticsearch {
               hosts => "localhost"
               index => "sales7"
               document_type => "order"
               template_name => "myindex"
               template => "/tmp/templates/custom-orders2.json"
               template_overwrite => true
     }

    stdout {}
}

JSON file

 {
  "template": "sales7",
  "settings": {
    "index.refresh_interval": "5s"
 },
"mappings": {
"sales": {
  "_source": {
    "enabled": false
  },
  "properties": {
    "location": {
      "type": "geo_point"
        }
      }
    }
  },
  "aliases": {}
  }
        index => "sales7"
               document_type => "order"
               template_name => "myindex"
               template => "/tmp/templates/custom-orders.json"
               template_overwrite => true
     }

    stdout {}
}

Interestingly, when the geo_point mapping doesn't work (ie. both lat and long are floats), my data is indexed (30 rows). But when the location is correctly made into a geo_point, none of my rows are indexed.

hkulekci hkulekci · Accepted Answer · 2017-07-23T13:41:12

There is two way to do this. First one is creating a template for your mapping to create a correct mapping while indexing you data. Because Elasticseach does not understand what your data type is. You should say it theses things like below.

Firstly, create a template.json file for your mapping structure:

{
  "template": "sales*",
  "settings": {
    "index.refresh_interval": "5s"
  },
  "mappings": {
    "sales": {
      "_source": {
        "enabled": false
      },
      "properties": {
        "location": {
          "type": "geo_point"
        }
      }
    }
  },
  "aliases": {}
}

After that change your logstash configuration to put this mapping your index :

input {
file {
  path => "/opt/logs/orders.log"
  start_position => "beginning"
 }
}

filter {
   grok {
       match => { "message" => "(?<date>[0-9-]+) (?<order_status>ORDER [a-zA-Z]+): (?<order_amount>£[0-9.]+) (?<order_location>[a-zA-Z ]+)"}
}

mutate {
       convert => { "order_amount" => "float" }
       convert => { "order_lat" => "float" }
       convert => { "order_long" => "float" }

       rename => {
                  "order_long" => "[location][lon]"
                  "order_lat" => "[location][lat]"
       }
 }
}

output {
      elasticsearch {
                hosts => "localhost"

                index => "sales"
                document_type => "order"
                template_name => "myindex"
                template => "/etc/logstash/conf.d/template.json"
                template_overwrite => true


     }
    stdout {}
}

Second option is ingest node feature. I will update my answer for this option but now you can check my dockerized repository. At this example, I used ingest node feature instead of template while parsing location data.

geo_point in Elastic

Update

1 Answers