Couldn't find any Elasticsearch data

0
votes

when I start kibana and looking in the management tab it is displaying a message

Couldn't find any Elasticsearch data
You'll need to index some data into Elasticsearch before you can create an index pattern

These are the data which are displayed in my elasticsearch cluster through the browser

http://localhost:9200/_cat/indices?v

health status index                          uuid                   pri rep docs.count docs.deleted store.size pri.store.size
green  open   .kibana-event-log-7.9.3-000001 JBL1C589TZWBEIjhe63f1w   1   0          3            0     16.2kb         16.2kb
green  open   .apm-custom-link               nqIKOV7rR8OhzG4Y6UntSA   1   0          0            0       208b           208b
green  open   .kibana_task_manager_1         3RbGcb5nTrelAfjr8cQ8Gg   1   0          6           38    150.3kb        150.3kb
green  open   .apm-agent-configuration       llIcAZAGTWanNghptfymVQ   1   0          0            0       208b           208b
green  open   .kibana_1                      tMQMj0UdRd-sCZPb631Y5g   1   0         23            9     10.4mb         10.4mb

I can not see logstash index and logstash is running under port 9600

http://localhost:9600/

{"host":"DESKTOP","version":"7.9.3","http_address":"127.0.0.1:9600","id":"b92c8d86-6159-4821-9ace-01bd5328f6af","name":"DESKTOP-MTG14LM","ephemeral_id":"4332a47b-ad63-4e02-a02e-5c233d7a3773","status":"green","snapshot":false,"pipeline":{"workers":4,"batch_size":125,"batch_delay":50},"build_date":"2020-10-16T12:25:47Z","build_sha":"d296f0087bdce367c37596241d5a1f00c9279193","build_snapshot":false}

logstash-sample.conf

input {
  file{
     type => "syslog"
     path => ["D:\Spring Boot Project\demo-gradle\another-log.log"]

}

output {

   stdout{
      codec => rubydebug
   }

  elasticsearch {
    hosts => ["http://localhost:9200"]
    index => "%{[@metadata][beat]}-%{[@metadata][version]}-%{+YYYY.MM.dd}"
    #user => "elastic"
    #password => "changeme"
  }
}

kindly support me to resolve this issue

enter image description here

http://localhost:9600/_node/stats/?pretty

{
  "host" : "DESKTOP",
  "version" : "7.9.3",
  "http_address" : "127.0.0.1:9600",
  "id" : "b92c8d86-6159-4821-9ace-01bd5328f6af",
  "name" : "DESKTOP",
  "ephemeral_id" : "15e38fba-b37a-4b7d-9e58-6a89e2082799",
  "status" : "green",
  "snapshot" : false,
  "pipeline" : {
    "workers" : 4,
    "batch_size" : 125,
    "batch_delay" : 50
  },
  "jvm" : {
    "threads" : {
      "count" : 30,
      "peak_count" : 32
    },
    "mem" : {
      "heap_used_percent" : 24,
      "heap_committed_in_bytes" : 1038876672,
      "heap_max_in_bytes" : 1038876672,
      "heap_used_in_bytes" : 255060960,
      "non_heap_used_in_bytes" : 174032152,
      "non_heap_committed_in_bytes" : 196833280,
      "pools" : {
        "old" : {
          "max_in_bytes" : 724828160,
          "peak_max_in_bytes" : 724828160,
          "used_in_bytes" : 125286040,
          "committed_in_bytes" : 724828160,
          "peak_used_in_bytes" : 226688920
        },
        "young" : {
          "max_in_bytes" : 279183360,
          "peak_max_in_bytes" : 279183360,
          "used_in_bytes" : 102941904,
          "committed_in_bytes" : 279183360,
          "peak_used_in_bytes" : 279183360
        },
        "survivor" : {
          "max_in_bytes" : 34865152,
          "peak_max_in_bytes" : 34865152,
          "used_in_bytes" : 26833016,
          "committed_in_bytes" : 34865152,
          "peak_used_in_bytes" : 34865144
        }
      }
    },
    "gc" : {
      "collectors" : {
        "old" : {
          "collection_time_in_millis" : 713,
          "collection_count" : 4
        },
        "young" : {
          "collection_time_in_millis" : 501,
          "collection_count" : 8
        }
      }
    },
    "uptime_in_millis" : 815971
  },
  "process" : {
    "open_file_descriptors" : -1,
    "peak_open_file_descriptors" : -1,
    "max_file_descriptors" : -1,
    "mem" : {
      "total_virtual_in_bytes" : -1
    },
    "cpu" : {
      "total_in_millis" : -1,
      "percent" : -3,
      "load_average" : null
    }
  },
  "events" : {
    "in" : 0,
    "filtered" : 0,
    "out" : 0,
    "duration_in_millis" : 0,
    "queue_push_duration_in_millis" : 0
  },
  "pipelines" : {
    "main" : {
      "events" : {
        "queue_push_duration_in_millis" : 0,
        "out" : 0,
        "duration_in_millis" : 0,
        "in" : 0,
        "filtered" : 0
      },
      "plugins" : {
        "inputs" : [ {
          "id" : "09ae4aa0701a92b926aee6c9c0abef34b22fe75695ed89371fb40e0ce5666067",
          "name" : "file",
          "events" : {
            "queue_push_duration_in_millis" : 0,
            "out" : 0
          }
        } ],
        "codecs" : [ {
          "id" : "plain_09312af1-ced8-4a87-8be0-7425fe846651",
          "name" : "plain",
          "encode" : {
            "writes_in" : 0,
            "duration_in_millis" : 0
          },
          "decode" : {
            "out" : 0,
            "writes_in" : 0,
            "duration_in_millis" : 0
          }
        }, {
          "id" : "rubydebug_88397be3-dcbe-4553-a788-aa3d4474e141",
          "name" : "rubydebug",
          "encode" : {
            "writes_in" : 0,
            "duration_in_millis" : 3
          },
          "decode" : {
            "out" : 0,
            "writes_in" : 0,
            "duration_in_millis" : 0
          }
        }, {
          "id" : "plain_497bb40b-2eab-4852-a002-e2c7ee4d7ab3",
          "name" : "plain",
          "encode" : {
            "writes_in" : 0,
            "duration_in_millis" : 0
          },
          "decode" : {
            "out" : 0,
            "writes_in" : 0,
            "duration_in_millis" : 0
          }
        } ],
        "filters" : [ ],
        "outputs" : [ {
          "id" : "e48f703a97c1645df3afa1d1b8937faffe8a408694f8a6ba5be6bb23bed53001",
          "name" : "stdout",
          "events" : {
            "out" : 0,
            "in" : 0,
            "duration_in_millis" : 33
          }
        }, {
          "id" : "ad540803354821020198353da7d7314b73309c07babecea3df737a197017449a",
          "name" : "elasticsearch",
          "events" : {
            "out" : 0,
            "in" : 0,
            "duration_in_millis" : 4
          }
        } ]
      },
      "reloads" : {
        "failures" : 0,
        "successes" : 0,
        "last_success_timestamp" : null,
        "last_error" : null,
        "last_failure_timestamp" : null
      },
      "queue" : {
        "type" : "memory",
        "events_count" : 0,
        "queue_size_in_bytes" : 0,
        "max_queue_size_in_bytes" : 0
      },
      "hash" : "661080585b2691f01bac24b363c27f0cfc03a009fbb302424abe96cc1ae50fb5",
      "ephemeral_id" : "faf3face-77dc-455f-8632-1ff2e1ebdd7c"
    }
  },
  "reloads" : {
    "failures" : 0,
    "successes" : 0
  },
  "os" : { },
  "queue" : {
    "events_count" : 0
  }

bin/logstash --log.level debug

[2020-11-16T21:59:20,627][DEBUG][logstash.runner          ] monitoring.elasticsearch.hosts: ["http://localhost:9200"]
[2020-11-16T21:59:20,630][DEBUG][logstash.runner          ] monitoring.collection.interval: #<LogStash::Util::TimeValue:0xa362681 @duration=10, @time_unit=:second>
[2020-11-16T21:59:20,635][DEBUG][logstash.runner          ] monitoring.collection.timeout_interval: #<LogStash::Util::TimeValue:0x228ca300 @duration=10, @time_unit=:minute>
[2020-11-16T21:59:20,637][DEBUG][logstash.runner          ] monitoring.elasticsearch.username: "logstash_system"
[2020-11-16T21:59:20,639][DEBUG][logstash.runner          ] monitoring.elasticsearch.ssl.verification_mode: "certificate"
[2020-11-16T21:59:20,640][DEBUG][logstash.runner          ] monitoring.elasticsearch.sniffing: false
[2020-11-16T21:59:20,641][DEBUG][logstash.runner          ] monitoring.collection.pipeline.details.enabled: true
[2020-11-16T21:59:20,643][DEBUG][logstash.runner          ] monitoring.collection.config.enabled: true
[2020-11-16T21:59:20,644][DEBUG][logstash.runner          ] node.uuid: ""
[2020-11-16T21:59:20,645][DEBUG][logstash.runner          ] --------------- Logstash Settings -------------------
[2020-11-16T21:59:20,711][DEBUG][logstash.config.source.multilocal] Reading pipeline configurations from YAML {:location=>"D:/ELK stack/logstash/config/pipelines.yml"}
ERROR: Pipelines YAML file is empty. Location: D:/ELK stack/logstash/config/pipelines.yml
usage:
  bin/logstash -f CONFIG_PATH [-t] [-r] [] [-w COUNT] [-l LOG]
  bin/logstash --modules MODULE_NAME [-M "MODULE_NAME.var.PLUGIN_TYPE.PLUGIN_NAME.VARIABLE_NAME=VALUE"] [-t] [-w COUNT] [-l LOG]
  bin/logstash -e CONFIG_STR [-t] [--log.level fatal|error|warn|info|debug|trace] [-w COUNT] [-l LOG]
  bin/logstash -i SHELL [--log.level fatal|error|warn|info|debug|trace]
  bin/logstash -V [--log.level fatal|error|warn|info|debug|trace]
  bin/logstash --help
[2020-11-16T21:59:20,755][ERROR][org.logstash.Logstash    ] java.lang.IllegalStateException: Logstash stopped processing because of an error: (SystemExit) exit

bin/logstash --log.level debug -f config/logstash-sample.conf

[2020-11-16T22:11:31,227][DEBUG][filewatch.sincedbcollection][main][09ae4aa0701a92b926aee6c9c0abef34b22fe75695ed89371fb40e0ce5666067] writing sincedb (delta since last write = 15)
[2020-11-16T22:11:32,314][DEBUG][logstash.instrument.periodicpoller.cgroup] One or more required cgroup files or directories not found: /proc/self/cgroup, /sys/fs/cgroup/cpuacct, /sys/fs/cgroup/cpu
[2020-11-16T22:11:32,678][DEBUG][logstash.instrument.periodicpoller.jvm] collector name {:name=>"ParNew"}
[2020-11-16T22:11:32,679][DEBUG][logstash.instrument.periodicpoller.jvm] collector name {:name=>"ConcurrentMarkSweep"}
[2020-11-16T22:11:34,964][DEBUG][org.logstash.execution.PeriodicFlush][main] Pushing flush onto pipeline.
[2020-11-16T22:11:37,330][DEBUG][logstash.instrument.periodicpoller.cgroup] One or more required cgroup files or directories not found: /proc/self/cgroup, /sys/fs/cgroup/cpuacct, /sys/fs/cgroup/cpu
[2020-11-16T22:11:37,691][DEBUG][logstash.instrument.periodicpoller.jvm] collector name {:name=>"ParNew"}
[2020-11-16T22:11:37,692][DEBUG][logstash.instrument.periodicpoller.jvm] collector name {:name=>"ConcurrentMarkSweep"}
[2020-11-16T22:11:39,964][DEBUG][org.logstash.execution.PeriodicFlush][main] Pushing flush onto pipeline.
[2020-11-16T22:11:42,336][DEBUG][logstash.instrument.periodicpoller.cgroup] One or more required cgroup files or directories not found: /proc/self/cgroup, /sys/fs/cgroup/cpuacct, /sys/fs/cgroup/cpu
[2020-11-16T22:11:42,697][DEBUG][logstash.instrument.periodicpoller.jvm] collector name {:name=>"ParNew"}
[2020-11-16T22:11:42,697][DEBUG][logstash.instrument.periodicpoller.jvm] collector name {:name=>"ConcurrentMarkSweep"}
[2020-11-16T22:11:44,960][DEBUG][org.logstash.execution.PeriodicFlush][main] Pushing flush onto pipeline.
2
spring-bootelasticsearchlogstashkibanakibana-7
Can you add to your question the response you get when hitting http://localhost:9600/_node/stats/?pretty ? - Val
localhost:9600/_node/stats/?pretty response updated - NafazBenzema
Ok, there are no events flowing through your pipeline... Are you sure that your config is correct? It's missing a closing } in your input - Val
I am a beginner and this is the first time I have tried ELK stack. I didn't modify any config files in logstash except logstash-sample.conf which is mentioned in the above - NafazBenzema
Can you run logstash with --debug on the command line and update your question with the output dumped during the start up? - Val

2 Answers

1
votes

Your file input is reading from the end of the file. Try modifying your input like this (i.e. read from the beginning):

input {
  file{
     type => "syslog"
     path => ["D:/Spring Boot Project/demo-gradle/another-log.log"]
     start_position => "beginning"
  }
}

And also make sure to remove this file:

D:/ELK stack/logstash/data/plugins/inputs/file/.sincedb_f8a2243b9184e26704b40adf1d7ef6af
0
votes

Thank You Val for your answer. In my case, windows has been used as an OS. So the backslash which is used to route the log path should be replaced with the forward slash.

logstash-sample.conf

input {
  file{
     path => [ "D:/Spring Boot Project/demo-gradle/another-log.log" ]
     start_position => "beginning"
  }
     
}

output {
   
   stdout{
      codec => rubydebug
   }

  elasticsearch {
    hosts => ["http://localhost:9200"]   
  }
}