I'm trying to deploy and ELK stack on AKS that will take messages from RabbitMQ and ultimately end up in Kibana. To do this I'm using the Elastic operator via
kubectl apply -f https://download.elastic.co/downloads/eck/1.3.0/all-in-one.yaml
Everything is working except the connection between Logstash and Elasticsearch. I can log in to Kibana, I can get the default Elasticsearch message in the browser, all the logs look fine so I think the issue lies in the logstash configuration. My configuration is at the end of the question, you can see I'm using secrets to get the various passwords and access the public certificates to make the https work.
Most confusingly, I can bash into the running logstash pod and with the exact same certificate run
curl --cacert /etc/logstash/certificates/tls.crt -u elastic:<redacted-password> https://rabt-db-es-http:9200
This gives me the response:
{
"name" : "rabt-db-es-default-0",
"cluster_name" : "rabt-db",
"cluster_uuid" : "9YoWLsnMTwq5Yor1ak2JGw",
"version" : {
"number" : "7.10.0",
"build_flavor" : "default",
"build_type" : "docker",
"build_hash" : "51e9d6f22758d0374a0f3f5c6e8f3a7997850f96",
"build_date" : "2020-11-09T21:30:33.964949Z",
"build_snapshot" : false,
"lucene_version" : "8.7.0",
"minimum_wire_compatibility_version" : "6.8.0",
"minimum_index_compatibility_version" : "6.0.0-beta1"
},
"tagline" : "You Know, for Search"
}
To me, this verifies that I the pod can communicate with the database, and has the correct user, password and certificates in place to do it securely. Why then does this fail using a logstash conf file?
The error from the logstash end is
[WARN ] 2021-01-14 15:24:38.360 [Ruby-0-Thread-6: /usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-output-elasticsearch-10.6.2-java/lib/logstash/outputs/elasticsearch/http_client/pool.rb:241] elasticsearch - Attempted to resurrect connection to dead ES instance, but got an error. {:url=>"https://rabt-db-es-http:9200/", :error_type=>LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError, :error=>"Got response code '401' contacting Elasticsearch at URL 'https://rabt-db-es-http:9200/'"}
From Elasticsearch I can see the failed requests as
{"type": "server", "timestamp": "2021-01-14T15:36:13,725Z", "level": "WARN", "component": "o.e.x.s.t.n.SecurityNetty4HttpServerTransport", "cluster.name": "rabt-db", "node.name": "rabt-db-es-default-0", "message": "http client did not trust this server's certificate, closing connection Netty4HttpChannel{localAddress=/10.244.0.30:9200, remoteAddress=/10.244.0.15:37766}", "cluster.uuid": "9YoWLsnMTwq5Yor1ak2JGw", "node.id": "9w3fXZBZQGeBMeFYGqYUXw" }
---
apiVersion: v1
kind: ConfigMap
metadata:
name: logstash-config
labels:
app.kubernetes.io/name: rabt
app.kubernetes.io/component: logstash
data:
logstash.yml: |
http.host: "0.0.0.0"
path.config: /usr/share/logstash/pipeline
---
apiVersion: v1
kind: ConfigMap
metadata:
name: logstash-pipeline
labels:
app.kubernetes.io/name: rabt
app.kubernetes.io/component: logstash
data:
logstash.conf: |
input {
rabbitmq {
host => "rabt-mq"
port => 5672
durable => true
queue => "rabt-rainfall-queue"
exchange => "rabt-rainfall-exchange"
exchange_type => "direct"
heartbeat => 30
durable => true
user => "${RMQ_USERNAME}"
password => "${RMQ_PASSWORD}"
}
file {
path => "/usr/share/logstash/config/intensity.csv"
start_position => "beginning"
codec => plain {
charset => "ISO-8859-1"
}
type => "intensity"
}
}
filter {
csv {
separator => ","
columns => ["Duration", "Intensity"]
}
}
output {
if [type] == "rainfall" {
elasticsearch {
hosts => [ "${ES_HOSTS}" ]
ssl => true
cacert => "/etc/logstash/certificates/tls.crt"
index => "rabt-rainfall-%{+YYYY.MM.dd}"
}
}
else if[type] == "intensity"{
elasticsearch {
hosts => [ "${ES_HOSTS}" ]
ssl => true
cacert => "/etc/logstash/certificates/tls.crt"
index => "intensity-%{+YYYY.MM.dd}"
}
}
}
---
apiVersion: v1
kind: ConfigMap
metadata:
name: rainfall-intensity-threshold
labels:
app.kubernetes.io/name: rabt
app.kubernetes.io/component: logstash
data:
intensity.csv: |
Duration,Intensity
0.1,7.18941593
0.2,6.34611898
0.3,5.89945352
0.4,5.60173824
0.5,5.38119846
0.6,5.20746530
0.7,5.06495933
0.8,4.94467113
0.9,4.84094288
1,4.75000000
2,4.19283923
3,3.89773029
4,3.70103175
5,3.55532256
6,3.44053820
7,3.34638544
8,3.26691182
9,3.19837924
10,3.13829388
20,2.77018141
30,2.57520486
40,2.44524743
50,2.34897832
60,2.27314105
70,2.21093494
80,2.15842723
90,2.11314821
100,2.07345020
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: rabt-logstash
labels:
app.kubernetes.io/name: rabt
app.kubernetes.io/component: logstash
spec:
replicas: 1
selector:
matchLabels:
app.kubernetes.io/name: rabt
app.kubernetes.io/component: logstash
template:
metadata:
labels:
app.kubernetes.io/name: rabt
app.kubernetes.io/component: logstash
spec:
containers:
- name: logstash
image: docker.elastic.co/logstash/logstash:7.9.2
ports:
- name: "tcp-beats"
containerPort: 5044
env:
- name: ES_HOSTS
value: "https://rabt-db-es-http:9200"
- name: ES_USER
value: "elastic"
- name: ES_PASSWORD
valueFrom:
secretKeyRef:
name: rabt-db-es-elastic-user
key: elastic
- name: RMQ_USERNAME
valueFrom:
secretKeyRef:
name: rabt-mq-default-user
key: username
- name: RMQ_PASSWORD
valueFrom:
secretKeyRef:
name: rabt-mq-default-user
key: password
volumeMounts:
- name: config-volume
mountPath: /usr/share/logstash/config
- name: pipeline-volume
mountPath: /usr/share/logstash/pipeline
- name: ca-certs
mountPath: /etc/logstash/certificates
readOnly: true
volumes:
- name: config-volume
projected:
sources:
- configMap:
name: logstash-config
- configMap:
name: rainfall-intensity-threshold
- name: pipeline-volume
configMap:
name: logstash-pipeline
- name: ca-certs
secret:
secretName: rabt-db-es-http-certs-public
---
apiVersion: v1
kind: Service
metadata:
name: rabt-logstash
labels:
app.kubernetes.io/name: rabt
app.kubernetes.io/component: logstash
spec:
ports:
- name: "tcp-beats"
port: 5044
targetPort: 5044
selector:
app.kubernetes.io/name: rabt
app.kubernetes.io/component: logstash
