Collect tomcat logs from tomcat docker container to Filebeat docker container

Question

I have a Tomcat docker container and Filebeat docker container both are up and running.

My objective: I need to collect tomcat logs from running Tomcat container to Filebeat container.

Issue: I have no idea how to get collected log files from Tomcat container.

What I have tried so far: I have tried to create a docker volume and add tomcat logs to that volume and access that volume from filebeat container, but ended with no success.

Structure: I have wrote docker-compose.yml file under project Logstash(root directory of the project) with following project structure.(Here I want to up and run Elasticsearch, Logstash, Filebeat and Kibana docker containers from one configuration file). docker-containers(root directory of the project) with following structure (here I want to up and run Tomcat, Nginx and Postgres containers from one configuration file).

• Logstash: contain 4 main sub directories (Filebeat, Logstash, Elasticsearch and Kibana), ENV file and docker-compose.yml file. Both sub directories contain Dockerfiles to pull images and build the containers.

• docker-containers: contains 3 main sub directories (Tomcat, Nginx and Postgres). ENV file and docker-compose.yml file. Both sub directories contain separate Dockerfiles to pull docker image and build the container.

• Note: I think this basic structure my helpful to understand my requirements.

docker-compose.yml files

Logstash.docker-compose.yml file

version: '2'
services:


  elasticsearch:
    container_name: OTP-Elasticsearch
    build:
      context: ./elasticsearch
      args:
        - ELK_VERSION=${ELK_VERSION}
    volumes:
      - ./elasticsearch/config/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml:ro
    ports:
      - "9200:9200"
      - "9300:9300"
    environment:
      ES_JAVA_OPTS: "-Xmx256m -Xms256m"
    networks:
      - elk

  filebeat:
    container_name: OTP-Filebeat
    command:
      - "-e"
      - "--strict.perms=false"
    user: root
    build:
      context: ./filebeat
      args:
        - ELK_VERSION=${ELK_VERSION}
    volumes:
      - ./filebeat/config/filebeat.yml:/usr/share/filebeat/filebeat.yml
    environment:
      LS_JAVA_OPTS: "-Xmx256m -Xms256m"
    networks:
      - elk
    depends_on: 
      - elasticsearch
      - logstash

  logstash:
    container_name: OTP-Logstash
    build:
      context: ./logstash
      args:
        - ELK_VERSION=${ELK_VERSION}
    volumes:
      - ./logstash/config/logstash.yml:/usr/share/logstash/config/logstash.yml:ro
      - ./logstash/pipeline:/usr/share/logstash/pipeline:ro
    expose:
      - 5044/tcp
    ports:
      - "9600:9600"
      - "5044:5044"
    environment:
      LS_JAVA_OPTS: "-Xmx256m -Xms256m"
    networks:
      - elk
    links:
      - elasticsearch
    depends_on:
      - elasticsearch


  kibana:
    container_name: OTP-Kibana
    build:
      context: ./kibana
      args:
        - ELK_VERSION=${ELK_VERSION}
    volumes:
      - ./kibana/config/:/usr/share/kibana/config:ro
    ports:
      - "5601:5601"
    networks:
      - elk
    links:
      - elasticsearch
    depends_on: 
      - elasticsearch
      - logstash
      - filebeat

networks:
  elk:
    driver: bridge

docker-containers.docker-compose.yml file

version: '2'
services:

  # Nginx
  nginx:
    container_name: OTP-Nginx
    restart: always
    build: 
      context: ./nginx
      args:
        - comapanycode=${COMPANY_CODE}
        - dbtype=${DB_TYPE}
        - dbip=${DB_IP}
        - dbname=${DB_NAME}
        - dbuser=${DB_USER}
        - dbpassword=${DB_PASSWORD}
        - webdirectory=${WEB_DIRECTORY}
    ports:
      - "80:80"
    links:
      - db:db
    volumes:
      - ./log/nginx:/var/log/nginx
    depends_on:
      - db

  # Postgres
  db:
    container_name: OTP-Postgres
    restart: always
    ports:
      - "5430:5430"
    build: 
      context: ./postgres
      args:
        - food_db_version=${FOOD_DB_VERSION}
        - dbtype=${DB_TYPE} 
        - retail_db_version=${RETAIL_DB_VERSION}
        - dbname=${DB_NAME} 
        - dbuser=${DB_USER}
        - dbpassword=${DB_PASSWORD}
    volumes:
      - .data/db:/octopus_docker/postgresql/data

  # Tomcat
  tomcat:
    container_name: OTP-Tomcat
    restart: always
    build: 
      context: ./tomcat
      args:
        - dbuser=${DB_USER}
        - dbpassword=${DB_PASSWORD}
    links:
      - db:db
    volumes:
      - ./tomcat/${WARNAME}.war:/usr/local/tomcat/webapps/${WARNAME}.war
    ports:
      - "8080:8080"
    depends_on:
      - db
      - nginx 

Additional files:

filebeat.yml (configuration file inside Logstash/Filbeat/config/)

filebeat.inputs:
- type: log
  enabled: true
  paths:
    - /usr/local/tomcat/logs/.*log
output.logstash:
  hosts: ["logstash:5044"]

Additional Info:

• System I am using is Ubuntu 18.04
• My goal is to collect tomcat logs from running tomcat container and forward them to Logstash and filter logs and forward that logs to Elasticsearch and finally to Kibana for Visualization purpose.
• For now I can collect local machine(host) logs and visualize them in Kibana.(/var/log/)

My Problem:

• I need to know proper way to get collected tomcat logs from tomcat container and forward them to logstash container via filebeat container.

Any discussion, answer or any help to understand a way to do this is highly expected.

Thanks.

Answer 1

So loooong... Create shared volume among all containers and setup your tomcat to save log files into that folder. If you can put all services into one docker-compose.yml, just setup volume internally:

docker-compose.yml

version: '3'
services:
  one:
    ...
    volumes:
      - logs:/var/log/shared
  two:
    ...
    volumes:
      - logs:/var/log/shared
volumes:
  logs:

If you need several docker-compose.yml files, create volume globally in advance with docker volume create logs and map it into both compose files:

version: '3'
services:
  one:
    ...
    volumes:
      - logs:/var/log/shared
  two:
    ...
    volumes:
      - logs:/var/log/shared
volumes:
  logs:
    external: true