Linux一键部署ELK+Filebeat+Nginx+Redis日志平台自动化脚本-武穆逸仙 In September 2024

环境准备

操作系统：CentOS Linux release 7.8.2003

软件版本

Elasticsearch：elasticsearch-7.5.1-linux-x86_64.tar.gz

Kibana：kibana-7.5.1-linux-x86_64.tar.gz

Logstash：logstash-7.5.1.tar.gz

Filebeat：filebeat-7.5.1-linux-x86_64.tar.gz

JDK：jdk-11.0.1_linux-x64_bin.tar.gz

Nginx：nginx-1.18.0.tar.gz

Redis：redis-5.0.7.tar.gz

脚本功能

1）一键安装 Elasticsearch、Kibana、Logstash、Filebeat

2）一键安装 Redis

3）一键安装 Nginx

4）自动添加 nginx_access、nginx_error 索引

5）自动配置 Elasticsearch 用户密码

[root@localhost ~]# vim install_elk_filebeat_redis.sh


#!/bin/bash
User="esuser"
Elasticsearch_User="esuser"
Elasticsearch_Passwd="ElasticPassword"
IPADDR=$(hostname -I |awk '{print $1}')
Elasticsearch_DIR="/elkdata/elasticsearch"
Kafka_IP=$(hostname -I |awk '{print $1}')
Zookeeper_IP=$(hostname -I |awk '{print $1}')
Elasticsearch_IP=$(hostname -I |awk '{print $1}')
# Define JDK path variables
JDK_URL=https://scimg.chem960.com/files
JDK_File=jdk-11.0.19_linux-x64_bin.tar.gz
JDK_File_Dir=jdk-11.0.19
JDK_Dir=/elkdata/jdk-11.0.19
# Define Redis path variables
Redis_URL=http://download.redis.io/releases
Redis_File=redis-5.0.9.tar.gz
Redis_File_Dir=redis-5.0.9
Redis_Prefix=/elkdata/redis
# Define Nginx path variables
Nginx_URL=http://nginx.org/download
Nginx_File=nginx-1.27.0.tar.gz
Nginx_File_Dir=nginx-1.27.0
Nginx_Dir=/elkdata/nginx
# Define Elasticsearch path variables
Elasticsearch_URL=https://artifacts.elastic.co/downloads/elasticsearch
Elasticsearch_File=elasticsearch-7.17.23-linux-x86_64.tar.gz
Elasticsearch_File_Dir=elasticsearch-7.17.23
Elasticsearch_Dir=/elkdata/elasticsearch
# Define Logstash path variables
Filebeat_URL=https://artifacts.elastic.co/downloads/beats/filebeat
Filebeat_File=filebeat-7.17.23-linux-x86_64.tar.gz
Filebeat_File_Dir=filebeat-7.17.23-linux-x86_64
Filebeat_Dir=/elkdata/filebeat
# Define Kafka path variables
Logstash_URL=https://artifacts.elastic.co/downloads/logstash
Logstash_File=logstash-7.17.23.tar.gz
Logstash_File_Dir=logstash-7.17.23
Logstash_Dir=/elkdata/logstash
# Define Kibana path variables
Kibana_URL=https://artifacts.elastic.co/downloads/kibana
Kibana_File=kibana-7.17.23-linux-x86_64.tar.gz
Kibana_File_Dir=kibana-7.17.23-linux-x86_64
Kibana_Dir=/elkdata/kibana
# 配置内核参数
cat >>/etc/security/limits.conf <>/etc/security/limits.d/20-nproc.conf </etc/sysctl.conf </dev/null
# 创建 elk 用户
[ $(grep -wc "elk" /etc/passwd) -eq 0 ] && useradd elk >/dev/null
# 安装 JDK 环境
java -version >/dev/null 2>&1
if [ $? -ne 0 ];then
	# Install Package
	[ -f /usr/bin/wget ] || yum -y install wget >/dev/null
	wget -c ${JDK_URL}/${JDK_File}
	tar xf ${JDK_File}
	mv ${JDK_File_Dir} ${JDK_Dir}
cat >>/etc/profile </dev/null
# Install Redis
if [ ! -d ${Redis_Prefix} ];then
	[ -f /usr/bin/openssl ] || yum -y install openssl openssl-devel
	yum -y install wget gcc gcc-c++
	wget -c ${Redis_URL}/${Redis_File}
	tar zxf ${Redis_File}
	\mv ${Redis_File_Dir} ${Redis_Prefix}
	cd ${Redis_Prefix} && make
	if [ $? -eq 0 ];then
		echo -e "\033[32mThe Redis Install Success...\033[0m"
	else
		echo -e "\033[31mThe Redis Install Failed...\033[0m"
	fi
else
	echo -e "\033[31mThe Redis has been installed...\033[0m"
	exit 1
fi
# 随机生成密码
Passwd=$(openssl rand -hex 12)
# Config Redis
ln -sf ${Redis_Prefix}/src/redis-* /usr/bin
sed -i "s/127.0.0.1/0.0.0.0/g" ${Redis_Prefix}/redis.conf
sed -i "/daemonize/s/no/yes/" ${Redis_Prefix}/redis.conf
sed -i "s/dir .*/dir \/data\/redis/" ${Redis_Prefix}/redis.conf
sed -i "s/logfile .*/logfile \/usr\/local\/redis\/redis.log/" ${Redis_Prefix}/redis.conf
sed -i '/appendonly/s/no/yes/' ${Redis_Prefix}/redis.conf
sed -i "s/# requirepass foobared/requirepass ${Passwd}/" ${Redis_Prefix}/redis.conf
echo never > /sys/kernel/mm/transparent_hugepage/enabled
sysctl vm.overcommit_memory=1
# Create data directory 
[ -d /data/redis ] || mkdir -p /data/redis
# 创建 systemctl 管理配置文件
cat >/usr/lib/systemd/system/redis.service </dev/null
	cd ~ && wget -c ${Elasticsearch_URL}/${Elasticsearch_File}
	tar xf ${Elasticsearch_File}
	mv ${Elasticsearch_File_Dir} ${Elasticsearch_Dir}
else
	echo -e "\033[31mThe Elasticsearch Already Install...\033[0m"
	exit 1
fi
# Install Kibana
if [ ! -d ${Kibana_Dir} ];then
	# Install Package
	[ -f /usr/bin/wget ] || yum -y install wget >/dev/null
	cd ~ && wget -c ${Kibana_URL}/${Kibana_File}
	tar xf ${Kibana_File}
	mv ${Kibana_File_Dir} ${Kibana_Dir}
else
	echo -e "\033[31mThe Kibana Already Install...\033[0m"
	exit 1
fi
# 配置 Elasticsearch
mkdir -p ${Elasticsearch_DIR}/{data,logs}
cat >${Elasticsearch_Dir}/config/elasticsearch.yml <${Kibana_Dir}/config/kibana.yml </dev/null && ${Elasticsearch_Dir}/bin/elasticsearch -d"
# 创建 systemctl 管理配置文件
cat >/usr/lib/systemd/system/elasticsearch.service </dev/null
	if [ $? -eq 0 ];then
		Code="break"
	fi
${Code}
done
# 生成 Elasticsearch 密码
cat >/tmp/config_elasticsearch_passwd.exp </dev/null
expect /tmp/config_elasticsearch_passwd.exp
# 创建 systemctl 管理配置文件
cat >/usr/lib/systemd/system/kibana.service <${Nginx_Dir}/conf/nginx.conf </usr/lib/systemd/system/nginx.service <${Filebeat_Dir}/filebeat.yml <${Logstash_Dir}/config/nginx.conf < "${IPADDR}"
        port => "6379"
        db => "0"
        password => "${Passwd}"
        data_type => "list"
        key => "all-access-log"
        codec => "json"
    }
}
filter {
    if [fields][logtype] == "nginx_access" {
        json {
            source => "message"
        } 
		
        grok {
            match => { "message" => "%{TIMESTAMP_ISO8601:timestamp} %{LOGLEVEL:level}" }
        }
		
        date {
            match => ["timestamp", "yyyy-MM-dd HH:mm:ss,SSS"]
            target => "@timestamp"
        }
    }
	if [fields][logtype] == "nginx_error" {
        json {
            source => "message"
        } 
		
        grok {
            match => { "message" => "%{TIMESTAMP_ISO8601:timestamp} %{LOGLEVEL:level}" }
        }
		
        date {
            match => ["timestamp", "yyyy-MM-dd HH:mm:ss,SSS"]
            target => "@timestamp"
        }
    }
}
output {
    if [fields][logtype] == "nginx_access" {
        elasticsearch {
            hosts => ["${Elasticsearch_IP}:9200"]
            user => "${Elasticsearch_User}"
            password => "${Elasticsearch_Passwd}"
            action => "index"
            index => "nginx_access.log-%{+YYYY.MM.dd}"
        }
    }
    if [fields][logtype] == "nginx_error" {
        elasticsearch {
            hosts => ["${Elasticsearch_IP}:9200"]
            user => "${Elasticsearch_User}"
            password => "${Elasticsearch_Passwd}"
            action => "index"
            index => "nginx_error.log-%{+YYYY.MM.dd}"
        }
    }	
}
EOF
# 创建 Filebeat 日志目录
[ -d ${Filebeat_Dir}/logs ] || mkdir ${Filebeat_Dir}/logs
# 授权 ELK 用户管理 Filebeat、Logstash
chown -R ${User}.${User} ${Filebeat_Dir}
chown -R ${User}.${User} ${Logstash_Dir}
# 启动 Filebeat
su ${User} -c "cd ${Filebeat_Dir} && nohup ./filebeat -e -c filebeat.yml >>${Filebeat_Dir}/logs/filebeat.log >/dev/null 2>&1 &"
# 启动 Logstash
su ${User} -c "cd ${Logstash_Dir}/bin && nohup ./logstash -f ${Logstash_Dir}/config/nginx.conf >/dev/null 2>&1 &"
# 判断 Logstash 服务是否启动,启动成功才执行以下操作
Code=""
while sleep 10
do
	echo -e "\033[32m$(date +'%F %T') 等待 Logstash 服务启动...\033[0m"
	# 获取 Logstash 服务端口
	netstat -lntup |grep "9600" >/dev/null
	if [ $? -eq 0 ];then
		Code="break"
	fi
${Code}
done
echo -e "\033[32mELK日志分析平台搭建完毕... \n 通过浏览器访问：http://${IPADDR}\n 用户名：${Elasticsearch_User}\n 密码：${Elasticsearch_Passwd}\033[0m"

脚本执行方式：

[root@localhost ~]# sh install_elk_filebeat_redis.sh

[root@localhost ~]# yum install -y wget && wget -O install.sh https://yun.iwmyx.cn/tools/elk.sh && sh elk.sh

脚本执行过程截图如下