docker 安装elk6.7.0

Docker►ES Docker, ES 评论

docker 安装elk6.7.0 logstash实现自动同步增量数据

  • 拉取容器并安装sebp/elk:670
1
docker run -it --privileged=true -p 5601:5601 -p 9200:9200  -p 5044:5044  -p 9300:9300 -v /opt/elasticsearch/elk-data:/var/lib/elasticsearch/data -v /opt/elasticsearch/plugins:/opt/elasticsearch/plugins  -v /opt/elasticsearch/logstash:/opt/logstash/config/config-mysql --name elk7 sebp/elk:670
1
cd /opt/elasticsearch/logstash/config/config-mysql

进入配置目录

下载mysql-connector-java-5.1.46.jar

新建goods.conf 和 goods_increment.conf

全量配置goods.conf

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
input {
   jdbc {
        jdbc_connection_string => "jdbc:mysql://172.17.0.1:3306/wl-produce-mall?characterEncoding=UTF-8&useSSL=false&autoReconnect=true"
        jdbc_user => "wltest"
        jdbc_password => "wltest"
        #此处的路径最好是绝对路径，行对路径取决与允许命令的目录
        jdbc_driver_library => "../config/config-mysql/mysql-connector-java-5.1.46.jar"
        jdbc_driver_class => "com.mysql.jdbc.Driver"
        jdbc_default_timezone =>"Asia/Shanghai"
	jdbc_paging_enabled => "true"
        jdbc_page_size => "50000"
        #此处的路径最好是绝对路径，行对路径取决与允许命令的目录
        #statement => "SELECT * FROM wp_goods WHERE  goods_status in (4,6) AND last_update_time > :sql_last_value AND last_update_time < NOW() ORDER BY last_update_time DESC"
        statement => "select * from wp_goods where goods_status in (4,6) "
		#statement => "SELECT * FROM wp_goods WHERE  goods_status IN (4,6) AND last_update_time > UNIX_TIMESTAMP( :sql_last_value )*1000 AND last_update_time < UNIX_TIMESTAMP(NOW())*1000 ORDER BY last_update_time DESC"
		#use_column_value => "true"
       	#tracking_column => "last_update_time"
       	#tracking_column_type => "numeric"
		#tracking_column_type => "timestamp"
		#record_last_run => "true"
	 	#last_run_metadata_path => "/opt/elk/other/config-mysql/lastquery"
		clean_run => "true"
		#statement_filepath => "/opt/elk/other/config-mysql/exec.sql"
#        schedule => "*/5 * * * * *"
		schedule => "*/2 * * * * *"
        type => "goods"
    }
}



output {
        
        #设置窗口日志输出
        stdout {
          codec => json_lines
        }

        if [type] == "sx_goods" {
                elasticsearch {
                hosts => ["localhost:9200"]
                #注意index的值不支持大写字母
                index => "sx_goods"
                #document_type自行设置，不设置时，默认为doc
                #document_type => "sx_goods"
                #此处的值来自查询sql中的列名称，根据需要自行配置
                document_id => "%{id}"
                template =>"../config/sx_goods/sx_goods_template.json"
                template_name =>"sx_goods"
                template_overwrite =>"true"
        }
    }
}

自增配置goods_increment.conf

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
input {
   jdbc {
        jdbc_connection_string => "jdbc:mysql://172.17.0.1:3306/wl-produce-mall?characterEncoding=UTF-8&useSSL=false&autoReconnect=true"
        jdbc_user => "wltest"
        jdbc_password => "wltest"
        #此处的路径最好是绝对路径，行对路径取决与允许命令的目录
        jdbc_driver_library => "../config/config-mysql/mysql-connector-java-5.1.46.jar"
        jdbc_driver_class => "com.mysql.jdbc.Driver"
        jdbc_default_timezone => "Asia/Shanghai"
	   	jdbc_paging_enabled => "true"
        jdbc_page_size => "50000"
        #此处的路径最好是绝对路径，行对路径取决与允许命令的目录
        #statement => "SELECT * FROM wp_goods WHERE  goods_status in (4,6) AND last_update_time > :sql_last_value AND last_update_time < NOW() ORDER BY last_update_time DESC"
        #statement => "select * from wp_goods where goods_status in (4,6) "
		statement => "SELECT * FROM wp_goods WHERE  goods_status IN (4,5,6,7) AND last_update_time > UNIX_TIMESTAMP( :sql_last_value )*1000 AND last_update_time < UNIX_TIMESTAMP(NOW())*1000 ORDER BY last_update_time DESC"
	#use_column_value => "true"
       #	tracking_column => "last_update_time"
       # tracking_column_type => "numeric"
	#tracking_column_type => "timestamp"
		#record_last_run => "true"
	 	#last_run_metadata_path => "/opt/elk/other/config-mysql/lastquery"
		clean_run => "false" 
		#statement_filepath => "/opt/elk/other/config-mysql/exec.sql"
       # schedule => "* * * * *"
		schedule => "*/5 * * * * *"
        type => "goods"
    }
}

output {
        
        #设置窗口日志输出
        stdout {
          codec => json_lines
        }

        if [type] == "goods" {
                elasticsearch {
                hosts => ["localhost:9200"]
                #注意index的值不支持大写字母
                index => "goods"
                #document_type自行设置，不设置时，默认为doc
                #document_type => "goods"
                #此处的值来自查询sql中的列名称，根据需要自行配置
                document_id => "%{id}"
        }
    }
}

注意坑：在容器中如果访问宿主ip数据库使用：172.17.0.1 172.17.0.2

先执行goods.conf导入所有商品数据

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
命令：
docker ps -a 

进入容器：
docker exec -it 容器id或名称 /bin/bash

cd /opt/logstash/bin

./logstash -f ../config/config-mysql/goods.conf 

成功后 ctrl+c 停止

nohub ./logstash -f ../config/config-mysql/goods_increment.conf  >/dev/null &

如果已经有执行的logstash任务可以加一个参数：--path.data=/config/config-mysql

查看日志
tailf /opt/elasticsearch/logstash/conf/nohup.out

# 安装参数
# 单例
-e "discovery.type=single-node"
# 启动的内存设置
-e ES_JAVA_OPTS="-Xms64m -Xmx512m" 

# 单独安装Es
docker run -it -d --privileged=true -p 9200:9200  -p 9300:9300  -v /opt/elasticsearch/elk-data:/var/lib/elasticsearch/data -e "discovery.type=single-node" -e ES_JAVA_OPTS="-Xms64m -Xmx512m"  --name elk670 elasticsearch:6.7.0
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
指定单个字段查询AND OR 可切换
POST /nba/_search
{
	"query": {
		"query_string": {
			"default_field": "displayNameEn",
			"query": "james OR harden" 
		}
	},
	"size": 100
}


指定多个字段查询

POST /nba/_search
{
	"query": {
		"query_string": {
			"fields": [
				"displayNameEn",
				"teamNameEn"
			],
			"query": "James AND Rockets"
		}
	},
	"size": 100
}

Elon ChungJAVA PHP Web Developer

乐观自信爱生活；沉着冷静撸代码。