本篇內容介紹了“docker-compose怎么搭建 es/kibana/logstash elk”的有關知識，在實際案例的操作過程中，不少人都會遇到這樣的困境，接下來就讓小編帶領大家學習一下如何處理這些情況吧！希望大家仔細閱讀，能夠學有所成！

在和碩等地區，都構建了全面的區域性戰略布局，加強發展的系統性、市場前瞻性、產品創新能力，以專注、極致的服務理念，為客戶提供成都網站制作、做網站 網站設計制作定制網站建設,公司網站建設,企業網站建設,成都品牌網站建設,成都全網營銷推廣,外貿營銷網站建設,和碩網站建設費用合理。

一、準備兩臺centos7 虛擬機

二、新建 /home/docker-contains/es   再創建node文件夾

分別在node下新建 /data  /logs   /conf文件

在conf目錄下 新建elasticsearch.yml文件如下

cluster.name: elasticsearch-cluster
node.name: es01
network.bind_host: 0.0.0.0
network.publish_host: 192.168.65.135
http.port: 9200
transport.tcp.port: 9300
http.cors.enabled: true
http.cors.allow-origin: "*"
node.master: true
node.data: true
discovery.zen.ping.unicast.hosts: ["192.168.65.135:9300","192.168.65.136:9300"]
discovery.zen.minimum_master_nodes: 1
path.logs: /usr/share/elasticsearch/logs
xpack.security.audit.enabled: true
~

回到es目錄：

新建docker-compose.yml如下

version: '3'
services:
     es01:
       image:  elasticsearch:6.6.1
       container_name: es01
       restart: always
       volumes:
         - /home/docker_container/es/master/data:/usr/share/elasticsearch/data:rw
         - /home/docker_container/es/master/conf/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml
         - /home/docker_container/es/master/logs:/user/share/elasticsearch/logs:rw
         - /home/docker_container/es/master/plugin1:/usr/share/elasticsearch/plugins:rw
       ports:
         - "9200:9200"
         - "9300:9300"

另外一臺機器重復上述過程：

新建elasticsearch.yml：

cluster.name: elasticsearch-cluster
node.name: es02
network.bind_host: 0.0.0.0
network.publish_host: 192.168.65.136
http.port: 9200
transport.tcp.port: 9300
http.cors.enabled: true
http.cors.allow-origin: "*"
node.master: true
node.data: true
discovery.zen.ping.unicast.hosts: ["192.168.65.135:9300","192.168.65.136:9300"]
discovery.zen.minimum_master_nodes: 1
path.logs: /usr/share/elasticsearch/logs
xpack.security.audit.enabled: true

新建docker-compose.yml:

version: '3'
services:
     es02:
       image:  elasticsearch:6.6.1
       container_name: es02
       restart: always
       volumes:
         - /home/docker-container/es/node1/data:/usr/share/elasticsearch/data:rw
         - /home/docker-container/es/node1/conf/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml
         - /home/docker-container/es/node1/logs:/user/share/elasticsearch/logs:rw
         - /home/docker-container/es/node1/plugin1:/usr/share/elasticsearch/plugins:rw
       ports:
         - "9200:9200"
         - "9300:9300"

分別在docker-compose.yml同級目錄執行 

docker-compose up -d

安裝kibana:

docker-compose.yml

version: '3'
services:
  kibana:
    image: kibana:6.6.1
    container_name: kibana
    volumes:
        - /home/docker_container/kibana/conf/kibana.yml:/usr/share/kibana/config/kibana.yml
    restart: always
    ports:
        - 5601:5601

kibana.yml:

elasticsearch.hosts: ["http://192.168.65.135:9200"]
server.host: "0.0.0.0"
xpack.monitoring.ui.container.elasticsearch.enabled: true
i18n.locale: zh-CN
~

如果出現  Kibana server is not ready yet

第一點：KB、ES版本不一致（網上大部分都是這么說的）

解決方法：把KB和ES版本調整為統一版本

第二點：kibana.yml中配置有問題（通過查看日志，發現了Error: No Living connections的問題）

解決方法：將配置文件kibana.yml中的elasticsearch.url改為正確的鏈接，默認為: http://elasticsearch:9200

改為http://自己的IP地址:9200

第三點：瀏覽器沒有緩過來

解決方法：刷新幾次瀏覽器（狂刷 我刷了 6遍才出來）。
 

logstash:

logstash采用非docker安裝，上傳logstash-6.4.3.tar.gz至/home/software目錄

解壓縮gz包，進入到解壓后的logstash-6.4.3目錄

編輯：vim config/pipelines.yml  （如果需要連接多個 只需在后面追加pipeline.id和path.config，切記pipeline.id不能重復）

#   Default is path.data/dead_letter_queue
#
#   path.dead_letter_queue:
- pipeline.id: table1
  path.config: "/home/software/logstash-6.4.3/conf/MySQL_1.conf"
- pipeline.id: table2
  path.config: "/home/software/logstash-6.4.3/conf/mysql.conf"

mysql.conf:

input {
  jdbc {
    jdbc_driver_library => "/home/software/logstash-6.4.3/sql/mysql-connector-java-5.1.46.jar"
    jdbc_driver_class => "com.mysql.jdbc.Driver"
    jdbc_connection_string => "jdbc:mysql://ip:3306/test"
    jdbc_user => "root"
    jdbc_password => ""
    schedule => "* * * * *"
    statement => "SELECT * FROM user WHERE update_time >= :sql_last_value"
    use_column_value => true
    tracking_column_type => "timestamp"
    tracking_column => "update_time"
    last_run_metadata_path => "syncpoint_table"
  }
}


output {
    elasticsearch {
        # ES的IP地址及端口
        hosts => ["192.168.65.135:9200","192.168.65.136:9200"]
        # 索引名稱 可自定義
        index => "user"
        # 需要關聯的數據庫中有有一個id字段，對應類型中的id
        document_id => "%{id}"
        document_type => "user"
    }
    stdout {
        # JSON格式輸出
        codec => json_lines
    }
}

mysql_1.conf:

input {
  jdbc {
    jdbc_driver_library => "/home/software/logstash-6.4.3/sql/mysql-connector-java-5.1.46.jar"
    jdbc_driver_class => "com.mysql.jdbc.Driver"
    jdbc_connection_string => "jdbc:mysql://ip:3306/lvz_goods?autoReconnect=true&useUnicode=true&createDatabaseIfNotExist=true&characterEncoding=utf8&useSSL=false&serverTimezone=GMT%2B8"
    jdbc_user => "root"
    jdbc_password => ""
    schedule => "* * * * *"
    statement => "SELECT * FROM lvz_product WHERE update_time >= :sql_last_value"
    use_column_value => true
    tracking_column_type => "timestamp"
    tracking_column => "update_time"
    last_run_metadata_path => "syncpoint_table"
  }
}


output {
    elasticsearch {
        # ES的IP地址及端口
        hosts => ["192.168.65.135:9200","192.168.65.136:9200"]
        # 索引名稱 可自定義
        index => "goods"
        # 需要關聯的數據庫中有有一個id字段，對應類型中的id
        document_id => "%{id}"
        document_type => "goods"
    }
    stdout {
        # JSON格式輸出
        codec => json_lines
    }
}

注意：需要把mysql連接包放到對應目錄；至此三大件安裝就完成了

es整合ik和拼音分詞器    1下載elasticsearch-analysis-ik-6.6.1.zip

分別上傳到兩臺機器上的node1和 node2目錄 ，解壓重命名為ik 

2下載elasticsearch-analysis-pinyin-6.6.1.zip
分別上傳到兩臺機器上的node1和 node2目錄 ，解壓重命名為pinyin

編輯docker-compose文件 掛載：

 /home/docker-container/es/node1/plugin1:/usr/share/elasticsearch/plugins:rw

重啟es，到kibana界面 ：

執行

先刪除goods索引

在goods索引自定義ik和拼音分詞器，ik_smart_pinyin：

DELETE  /goods
PUT /goods
{
   "settings": {
        "analysis": {
            "analyzer": {
                "ik_smart_pinyin": {
                    "type": "custom",
                    "tokenizer": "ik_smart",
                    "filter": ["my_pinyin", "word_delimiter"]
                },
                "ik_max_word_pinyin": {
                    "type": "custom",
                    "tokenizer": "ik_max_word",
                    "filter": ["my_pinyin", "word_delimiter"]
                }
            },
            "filter": {
                "my_pinyin": {
                    "type" : "pinyin",
                    "keep_separate_first_letter" : true,
                    "keep_full_pinyin" : true,
                    "keep_original" : true,
                    "limit_first_letter_length" : 16,
                    "lowercase" : true,
                    "remove_duplicated_term" : true 
                }
            }
        }
  }
  
}

指定goods索引為ik_smart_pinyin

POST /goods/_mapping/goods
{
   
  
      "goods": {
        "properties": {
          "@timestamp": {
            "type": "date"
          },
          "@version": {
            "type": "text",
            "fields": {
              "keyword": {
                "type": "keyword",
                "ignore_above": 256
              }
            }
          },
          "attribute_list": {
            "type": "text",
            "fields": {
              "keyword": {
                "type": "keyword",
                "ignore_above": 256
              }
            }
          },
          "category_id": {
            "type": "long"
          },
          "created_time": {
            "type": "date"
          },
          "detail": {
            "type": "text",
             "analyzer":"ik_smart_pinyin",
            "search_analyzer":"ik_smart_pinyin"

          },
          "id": {
            "type": "long"
          },
          "main_image": {
            "type": "text",
            "fields": {
              "keyword": {
                "type": "keyword",
                "ignore_above": 256
              }
            }
          },
          "name": {
            "type": "text",
            "analyzer":"ik_smart_pinyin",
            "search_analyzer":"ik_smart_pinyin"

          },
          "revision": {
            "type": "long"
          },
          "status": {
            "type": "long"
          },
          "sub_images": {
            "type": "text",
            "fields": {
              "keyword": {
                "type": "keyword",
                "ignore_above": 256
              }
            }
          },
          "subtitle": {
            "type": "text",
          "analyzer":"ik_smart",
         "search_analyzer":"ik_smart"

          },
          "updated_time": {
            "type": "date"
          }
        }
      }
}

安裝kafka:

version: '2'
services:
  zookeeper:
    image: wurstmeister/zookeeper
    ports:
      - "2181:2181"

  kafka:
    image: wurstmeister/kafka
    ports:
      - "9092"             # kafka 把9092端口隨機映射到主機的端口
    environment:
      KAFKA_ADVERTISED_HOST_NAME: 192.168.65.135           #本機ip
      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
      KAFKA_CREATE_TOPICS: test:1:1
    volumes:
      - /var/run/docker.sock:/var/run/docker.sock

啟動兩個節點 kafka:

docker-compose up -d --scale kafka=2        本機啟動一個有兩個節點的 Kafka 集群

本地項目集成，kafka和elk;

application.yml

###服務啟動端口號
server:
  port: 8500
###服務名稱(服務注冊到eureka名稱)
eureka:
  client:
    service-url:
      defaultZone: http://localhost:8100/eureka

spring:
  application:
    name:  app-lvz-goods
  redis:
    host: 192.168.65.136
    port: 6379
    password: feilvzhang
    pool:
      max-idle: 100
      min-idle: 1
      max-active: 1000
      max-wait: -1
  ###數據庫相關連接
  datasource:
    username: root
    password: 
    driver-class-name: com.mysql.jdbc.Driver
    url: jdbc:mysql://192.168.125.113:3306/lvz_goods?autoReconnect=true&useUnicode=true&createDatabaseIfNotExist=true&characterEncoding=utf8&useSSL=false&serverTimezone=GMT%2B8
  data:
    elasticsearch:
      ####集群名稱
      cluster-name: elasticsearch-cluster
      ####地址
      cluster-nodes: 192.168.65.135:9300,192.168.65.136:9300
  kafka:
    bootstrap-servers: 192.168.65.135:32768,192.168.65.135:32769

日志切面：

package com.lvz.shop.elk.aop;

import com.alibaba.fastjson.JSONObject;
import com.lvz.shop.elk.kafka.KafkaSender;
import org.aspectj.lang.JoinPoint;
import org.aspectj.lang.annotation.AfterReturning;
import org.aspectj.lang.annotation.Aspect;
import org.aspectj.lang.annotation.Before;
import org.aspectj.lang.annotation.Pointcut;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import org.springframework.web.context.request.RequestContextHolder;
import org.springframework.web.context.request.ServletRequestAttributes;

import javax.servlet.http.HttpServletRequest;
import java.text.SimpleDateFormat;
import java.util.Arrays;
import java.util.Date;


/**
 * @description: ELK攔截日志信息
 * @author: flz
 * @date: 2019/8/9 15:57
 */
@Aspect
@Component
public class AopLogAspect {
    @Autowired
    private KafkaSender kafkaSender;

    // 申明一個切點 里面是 execution表達式
    @Pointcut("execution(* com.lvz.shop.*.impl.*.*(..))")
    private void serviceAspect() {
    }

    // 請求method前打印內容
    @Before(value = "serviceAspect()")
    public void methodBefore(JoinPoint joinPoint) {
        ServletRequestAttributes requestAttributes = (ServletRequestAttributes) RequestContextHolder
                .getRequestAttributes();
        HttpServletRequest request = requestAttributes.getRequest();

        // // 打印請求內容
        // log.info("===============請求內容===============");
        // log.info("請求地址:" + request.getRequestURL().toString());
        // log.info("請求方式:" + request.getMethod());
        // log.info("請求類方法:" + joinPoint.getSignature());
        // log.info("請求類方法參數:" + Arrays.toString(joinPoint.getArgs()));
        // log.info("===============請求內容===============");

        JSONObject jsonObject = new JSONObject();
        SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");// 設置日期格式
        //請求時間
        jsonObject.put("request_time", df.format(new Date()));
        //請求URL
        jsonObject.put("request_url", request.getRequestURL().toString());
        //請求方法
        jsonObject.put("request_method", request.getMethod());
        //請求類方法
        jsonObject.put("signature", joinPoint.getSignature());
        //請求參數
        jsonObject.put("request_args", Arrays.toString(joinPoint.getArgs()));
        JSONObject requestJsonObject = new JSONObject();
        requestJsonObject.put("request", jsonObject);
        kafkaSender.send(requestJsonObject);

    }

    // 在方法執行完結后打印返回內容
    @AfterReturning(returning = "o", pointcut = "serviceAspect()")
    public void methodAfterReturing(Object o) {
        // log.info("--------------返回內容----------------");
        // log.info("Response內容:" + gson.toJson(o));
        // log.info("--------------返回內容----------------");
        JSONObject respJSONObject = new JSONObject();
        JSONObject jsonObject = new JSONObject();
        SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");// 設置日期格式
        jsonObject.put("response_time", df.format(new Date()));
        jsonObject.put("response_content", JSONObject.toJSONString(o));
        respJSONObject.put("response", jsonObject);
        kafkaSender.send(respJSONObject);

    }
}

kafka消息發送：

package com.lvz.shop.elk.kafka;

import com.alibaba.fastjson.JSON;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.kafka.core.KafkaTemplate;
import org.springframework.kafka.support.SendResult;
import org.springframework.stereotype.Component;
import org.springframework.util.concurrent.ListenableFuture;
import org.springframework.util.concurrent.ListenableFutureCallback;

/**
 * @description: 生產者
 * @author: flz
 * @date: 2019/8/9 15:59
 */
@Component
@Slf4j
public class KafkaSender {

    @Autowired
    private KafkaTemplate kafkaTemplate;

    /**
     * kafka 發送消息
     *
     * @param obj 消息對象
     */
    public void send(T obj) {
        String jsonObj = JSON.toJSONString(obj);
        log.info("------------ message = {}", jsonObj);

        // 發送消息
        ListenableFuture> future = kafkaTemplate.send("goods_mylog", jsonObj);
        future.addCallback(new ListenableFutureCallback>() {
            @Override
            public void onFailure(Throwable throwable) {
                log.info("Produce: The message failed to be sent:" + throwable.getMessage());
            }

            @Override
            public void onSuccess(SendResult stringObjectSendResult) {
                // TODO 業務處理
                log.info("Produce: The message was sent successfully:");
                log.info("Produce: _+_+_+_+_+_+_+ result: " + stringObjectSendResult.toString());
            }
        });
    }
}

異常切面日志：

package com.lvz.shop.elk.aop.error;

import com.alibaba.fastjson.JSONObject;
import com.lvz.shop.elk.kafka.KafkaSender;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.ControllerAdvice;
import org.springframework.web.bind.annotation.ExceptionHandler;
import org.springframework.web.bind.annotation.ResponseBody;

import java.text.SimpleDateFormat;
import java.util.Date;

/**
 * @description: 全局異常處理
 * @author: flz
 * @date: 2019/8/9 15:56
 */
//@ControllerAdvice
@Slf4j
public class GlobalExceptionHandler {
    @Autowired
    private KafkaSender kafkaSender;

    @ExceptionHandler(RuntimeException.class)
    @ResponseBody
    public JSONObject exceptionHandler(Exception e) {
        log.info("###全局捕獲異常###,error:{}", e);

        // 1.封裝異常日志信息
        JSONObject errorJson = new JSONObject();
        JSONObject logJson = new JSONObject();
        SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");// 設置日期格式
        logJson.put("request_time", df.format(new Date()));
        logJson.put("error_info", e);
        errorJson.put("request_error", logJson);
        kafkaSender.send(errorJson);
        // 2. 返回錯誤信息
        JSONObject result = new JSONObject();
        result.put("code", 500);
        result.put("msg", "系統錯誤");

        return result;
    }
}

logstash 配置kafka和es:

goods_mylog.conf:

input {
  kafka {
    bootstrap_servers => ["192.168.65.135:32768,192.168.65.135:32769"]
    topics => ["goods_mylog"]
  }
}
output {
    stdout { codec => rubydebug }
    elasticsearch {
       hosts => ["192.168.65.135:9200","192.168.65.136:9200"]
       index => "goods_mylog"
    }
}

mylog.conf:

input {
  kafka {
    bootstrap_servers => ["192.168.65.135:32768,192.168.65.135:32769"]
    topics => ["my_log"]
  }
}
output {
    stdout { codec => rubydebug }
    elasticsearch {
       hosts => ["192.168.65.135:9200","192.168.65.136:9200"]
       index => "my_log"
    }
}

啟動項目 訪問 接口elk就會開始收集日志，在kibana下查看：

“docker-compose怎么搭建 es/kibana/logstash elk”的內容就介紹到這里了，感謝大家的閱讀。如果想了解更多行業相關的知識可以關注創新互聯網站，小編將為大家輸出更多高質量的實用文章！