Zabbix 常用自定义监控项
系统监控常用自带监控项
- agent.ping -- 检测客户端可达性、返回nothing表示不可达。1表示可达
- system.cpu.load -- 检测cpu负载。返回浮点数
- system.cpu.util -- 检测cpu使用率。返回浮点数
- vfs.dev.read -- 检测硬盘读取数据,返回是sps.ops.bps浮点类型,需要定义1024倍
- vfs.dev.write -- 检测硬盘写入数据。返回是sps.ops.bps浮点类型,需要定义1024倍
- net.if.out[br0] --检测网卡流速、流出方向,时间间隔为60S
- net-if-in[br0] --检测网卡流速,流入方向(单位:字节) 时间间隔60S
- proc.num[] 目前系统中的进程总数,时间间隔60s
- proc.num[,,run] 目前正在运行的进程总数,时间间隔60S
- system.cpu.swtiches --cpu的进程上下文切换,单位sps,表示每秒采样次数,api中参数history需指定为3
- system.cpu.intr --cpu中断数量、api中参数history需指定为3
- system.cpu.load[percpu,avg1] --cpu每分钟的负载值,按照核数做平均值(Processor load (1 min average per core)),api中参数history需指定为0
- system.cpu.load[percpu,avg5] --cpu每5分钟的负载值,按照核数做平均值(Processor load (5 min average per core)),api中参数history需指定为0
- system.cpu.load[percpu,avg15] --cpu每5分钟的负载值,按照核数做平均值(Processor load (15 min average per core)),api中参数history需指定为0
系统监控常用自定义监控项
内存相关
bash
# 创建配置文件
cat > /etc/zabbix/zabbix_agent2.d/memory_advanced.conf << "EOF"
UserParameter=memory.rate[*],free -m | awk -v type='$1' '/^Mem/{ if (type == "used") { printf "%.2f", ($$3/$$2)*100 }; if (type == "free") { printf "%.2f", ($$7/$$2)*100 }; if (type == "cache") { printf "%.2f", ($$6/$$2)*100 }}'
EOF
# 在可视化界面中添加监控项
memory.rate[cache] --检测内存的缓存使用量,返回浮点数
memory.rate[free] --检测内存的空闲率,返回浮点数
memory.rate[used] --检测内存的使用率,返回浮点数Redis 相关的自定义项(未测试)
bash
vim /usr/local/zabbix/etc/zabbix_agentd.conf.d/redis.conf
UserParameter=Redis.Status,/usr/local/redis/bin/redis-cli -h 127.0.0.1 -p 6379 ping |grep -c PONG
UserParameter=Redis_conn[*],/usr/local/redis/bin/redis-cli -h $1 -p $2 info | grep -w "connected_clients" | awk -F':' '{print $2}'
UserParameter=Redis_rss_mem[*],/usr/local/redis/bin/redis-cli -h $1 -p $2 info | grep -w "used_memory_rss" | awk -F':' '{print $2}'
UserParameter=Redis_lua_mem[*],/usr/local/redis/bin/redis-cli -h $1 -p $2 info | grep -w "used_memory_lua" | awk -F':' '{print $2}'
UserParameter=Redis_cpu_sys[*],/usr/local/redis/bin/redis-cli -h $1 -p $2 info | grep -w "used_cpu_sys" | awk -F':' '{print $2}'
UserParameter=Redis_cpu_user[*],/usr/local/redis/bin/redis-cli -h $1 -p $2 info | grep -w "used_cpu_user" | awk -F':' '{print $2}'
UserParameter=Redis_cpu_sys_cline[*],/usr/local/redis/bin/redis-cli -h $1 -p $2 info | grep -w "used_cpu_sys_children" | awk -F':' '{print $2}'
UserParameter=Redis_cpu_user_cline[*],/usr/local/redis/bin/redis-cli -h $1 -p $2 info | grep -w "used_cpu_user_children" | awk -F':' '{print $2}'
UserParameter=Redis_keys_num[*],/usr/local/redis/bin/redis-cli -h $1 -p $2 info | grep -w "$$1" | grep -w "keys" | grep db$3 | awk -F'=' '{print $2}' | awk -F',' '{print $1}'
UserParameter=Redis_loading[*],/usr/local/redis/bin/redis-cli -h $1 -p $2 info | grep loading | awk -F':' '{print $$2}'
Redis.Status --检测Redis运行状态, 返回整数
Redis_conn --检测Redis成功连接数,返回整数
Redis_rss_mem --检测Redis系统分配内存,返回整数
Redis_lua_mem --检测Redis引擎消耗内存,返回整数
Redis_cpu_sys --检测Redis主程序核心CPU消耗率,返回整数
Redis_cpu_user --检测Redis主程序用户CPU消耗率,返回整数
Redis_cpu_sys_cline --检测Redis后台核心CPU消耗率,返回整数
Redis_cpu_user_cline --检测Redis后台用户CPU消耗率,返回整数
Redis_keys_num --检测库键值数,返回整数
Redis_loding --检测Redis持久化文件状态,返回整数Nginx 相关的自定义项
bash
# nginx 配置
vim /etc/nginx/conf.d/default.conf
location /nginxstatus
{
stub_status on;
access_log off;
allow 127.0.0.1;
deny all;
}
# 在命令行中使用curl -s http://127.0.0.1/nginxstatus即可获得nginx的连接状态。
# zabbix 监控
vim /usr/local/zabbix/etc/zabbix_agentd.conf.d/nginx.conf
UserParameter=Nginx.active,/usr/bin/curl -s "http://127.0.0.1:80/nginxstatus" | awk '/Active/ {print $NF}'
UserParameter=Nginx.read,/usr/bin/curl -s "http://127.0.0.1:80/nginxstatus" |awk '/Reading/{print $2}'
UserParameter=Nginx.wrie,/usr/bin/curl -s "http://127.0.0.1:80/nginxstatus" |awk '/Writing/{print $4}'
UserParameter=Nginx.wait,/usr/bin/curl -s "http://127.0.0.1:80/nginxstatus" |awk '/Waiting/{print $6}'
UserParameter=Nginx.accepted,/usr/bin/curl -s "http://127.0.0.1:80/nginxstatus" | awk '/^[ \t]+[0-9]+[ \t]+[0-9]+[ \t]+[0-9]+/ {print $1}'
UserParameter=Nginx.handled,/usr/bin/curl -s "http://127.0.0.1:80/nginxstatus" | awk '/^[ \t]+[0-9]+[ \t]+[0-9]+[ \t]+[0-9]+/ {print $2}'
UserParameter=Nginx.requests,/usr/bin/curl -s "http://127.0.0.1:80/nginxstatus" | awk '/^[ \t]+[0-9]+[ \t]+[0-9]+[ \t]+[0-9]+/ {print $3}'TCP相关的自定义项
bash
cat > /usr/local/zabbix/etc/zabbix_agentd.conf.d/tcp_connect.conf << "EOF"
UserParameter=socket.tcp_listen,grep -c "\b 0A \b" /proc/net/tcp
UserParameter=socket.tcp_closing,grep -c "\b 0B \b" /proc/net/tcp
UserParameter=socket.tcp_established,grep -c "\b 01 \b" /proc/net/tcp
UserParameter=socket.tcp_synsent,grep -c "\b 02 \b" /proc/net/tcp
UserParameter=socket.tcp_synrecv,grep -c "\b 03 \b" /proc/net/tcp
UserParameter=socket.tcp_finwait1,grep -c "\b 04 \b" /proc/net/tcp
UserParameter=socket.tcp_finwait2,grep -c "\b 05 \b" /proc/net/tcp
UserParameter=socket.tcp_timewait,grep -c "\b 06 \b" /proc/net/tcp
UserParameter=socket.tcp_closed,grep -c "\b 07 \b" /proc/net/tcp
UserParameter=socket.tcp_closewait,grep -c "\b 08 \b" /proc/net/tcp
UserParameter=socket.tcp_lastack,grep -c "\b 09 \b" /proc/net/tcp
EOF
systemctl restart zabbix-agent2
# 结果返回整数JMX 监控 Tomcat
官方文档:https://www.zabbix.com/documentation/6.4/zh/manual/config/items/itemtypes/jmx_monitoring
从zabbix 2.0开始,JMX监控端以Zabbix守护进程的形式运行,称为“Zabbix Java gateway”。
环境准备
| 主机 | IP | 安装服务 |
|---|---|---|
| zabbix-server | 192.168.148.105 | java-gateway |
| zabbix-agent | 192.168.148.101 | JDK、tomcat、JMX |
zabbix-agent配置
提前装好JDK和Tomcat
配置 JMX
bash
# 配置tomcat远程端口、加密,最好放在脚本文件的开头部分比如`# OS specific support. $var _must_ be set to either true or false.`的下一行
vim /usr/local/tomcat/bin/catalina.sh
CATALINA_OPTS='-Dcom.sun.management.jmxremote
-Dcom.sun.management.jmxremote.port=12345
-Dcom.sun.management.jmxremote.ssl=false
-Dcom.sun.management.jmxremote.authenticate=false
-Djava.rmi.server.hostname=192.168.148.101'zabbix-server配置
安装Zabbix Java gateway
bash
dnf install zabbix-java-gateway -y修改Zabbix Java gateway配置文件
bash
vim /etc/zabbix/zabbix_java_gateway.conf
# 参数值都存在,只需打开注释即可
LISTEN_IP="0.0.0.0" #监听地址
LISTEN_PORT=10052 #Java gateway默认监听10052端口
START_POLLERS=5 #开启的工作线程数
TIMEOUT=30 #超时时间
PID_FILE="/var/run/zabbix/zabbix_java.pid" #PID_FILE文件
# 验证
grep "^[a-Z]" /etc/zabbix/zabbix_java_gateway.conf启动并验证Zabbix Java gateway服务
bash
systemctl enable --now zabbix-java-gateway
systemctl status zabbix-java-gateway
ss -antup | grep 10052配置zabbix server调用java gateway
bash
cat >> /usr/local/zabbix/etc/zabbix_server.conf << "EOF"
JavaGateway=192.168.148.104
JavaGatewayPort=10052
StartJavaPollers=5
EOF
# 建议让StartJavaPollers选项的值小于或等于START_POLLERS
systemctl restart zabbix- JavaGateway:JavaGateway的所在服务器IP地址
- JavaGatewayPort:JavaGateway的服务端口
- StartJavaPollers:从javaGateway采集数据的进程数
自定义监控 Tomcat 状态
创建配置
bash
cat > /etc/zabbix/zabbix_agent2.d/tomcat_ststua.conf << "EOF"
UserParameter=tomcat.status,ss -antp | grep 8080 | awk '{print $1}'| grep -o "LISTEN" >/dev/null && echo 1 || echo 0
EOF重启agent2
bash
systemctl restart zabbix-agent2
systemctl status zabbix-agent2验证
bash
# 在zabbix-server执行
zabbix_get -s '192.168.148.101' -p 10050 -k 'tomcat.status'JMX监控项
JMX监控项添加主机页面选择好模板[Apache Tomcat by JMX]都是配置好的监控项,可以直接添加自定义监控图表查看即可。
下载jar包
获取jvm需要用到catalina-jmx-remote.jar和cmdline-jmxclient-0.10.3.jar两个包,根据在即tomcat版本下载,下载后放到tomcat的bin目录下。
bash
wget -O /usr/local/tomcat/bin/catalina-jmx-remote.jar http://archive.apache.org/dist/tomcat/tomcat-8/v8.0.27/bin/extras/catalina-jmx-remote.jar
# cmdline-jmxclient-0.10.3.jar下载地址
http://crawler.archive.org/cmdline-jmxclient/cmdline-jmxclient-0.10.3.jar
# 重启tomcat
# 测试
java -jar /usr/local/tomcat/bin/cmdline-jmxclient-0.10.3.jar - 127.0.0.1:12345 java.lang:type=Memory HeapMemoryUsage堆内存
bash
# 指令查看
java -jar /usr/local/tomcat/bin/cmdline-jmxclient-0.10.3.jar controlRole:tomcat 127.0.0.1:12345 java.lang:type=Memory HeapMemoryUsage
# Zabbix监控项的键值
jmx["java.lang:type=Memory","HeapMemoryUsage.committed"] # 已提交堆内存
jmx["java.lang:type=Memory","HeapMemoryUsage.max"] # 堆内存最大值
jmx["java.lang:type=Memory","HeapMemoryUsage.used"] # 已用堆内存内存池 Survivor Space
bash
# 指令查看
java -jar /usr/local/tomcat/bin/cmdline-jmxclient-0.10.3.jar - 127.0.0.1:12345 java.lang:name=Survivor\ Space,type=MemoryPool Usage
# Zabbix监控项的键值
jmx["java.lang:type=MemoryPool,name=Survivor Space",Usage.committed] # 已提交空间
jmx["java.lang:type=MemoryPool,name=Survivor Space",Usage.max] # 内存池最大空间
jmx["java.lang:type=MemoryPool,name=Survivor Space",Usage.used] # 内存池已用空间内存池 Eden Space
bash
# 指令查看
java -jar /usr/local/tomcat/bin/cmdline-jmxclient-0.10.3.jar - 127.0.0.1:12345 java.lang:name=Eden\ Space,type=MemoryPool Usage
# Zabbix监控项的键值
jmx["java.lang:type=MemoryPool,name=Eden Space",Usage.committed] # 已提交空间
jmx["java.lang:type=MemoryPool,name=Eden Space",Usage.max] # 内存池最大空间
jmx["java.lang:type=MemoryPool,name=Eden Space",Usage.used] # 内存池已用空间内存池 Old Gen
bash
# 指令查看
java -jar /usr/local/tomcat/bin/cmdline-jmxclient-0.10.3.jar - 127.0.0.1:12345 java.lang:name=Old\ Gen,type=MemoryPool Usage
# Zabbix监控项的键值
jmx["java.lang:type=MemoryPool,name=Old Gen",Usage.committed] # 已提交空间
jmx["java.lang:type=MemoryPool,name=Old Gen",Usage.max] # 内存池最大空间
jmx["java.lang:type=MemoryPool,name=Old Gen",Usage.used] # 内存池已用空间内存池 Meta Space
bash
# 指令查看
java -jar /usr/local/tomcat/bin/cmdline-jmxclient-0.10.3.jar - 127.0.0.1:12345 java.lang:type=MemoryPool,name=Metaspace Usage
# Zabbix监控项的键值
jmx["java.lang:type=MemoryPool,name=Metaspace",Usage.committed] # 已提交空间
jmx["java.lang:type=MemoryPool,name=Metaspace",Usage.used] # 内存池已用空间内存池 Code Cache
bash
# 指令查看
java -jar /usr/local/tomcat/bin/cmdline-jmxclient-0.10.3.jar - 127.0.0.1:12345 java.lang:type=MemoryPool,name=Code\ Cache Usage
# Zabbix监控项的键值
jmx["java.lang:type=MemoryPool,name=Code Cache",Usage.committed] # 已提交空间
jmx["java.lang:type=MemoryPool,name=Code Cache",Usage.max] # 内存池最大空间
jmx["java.lang:type=MemoryPool,name=Code Cache",Usage.used] # 内存池已用空间非堆内存
bash
# 指令查看
java -jar /usr/local/tomcat/bin/cmdline-jmxclient-0.10.3.jar - 127.0.0.1:12345 java.lang:type=Memory NonHeapMemoryUsage
# Zabbix监控项的键值
jmx["java.lang:type=Memory","NonHeapMemoryUsage.committed"] # 已提交空间
jmx["java.lang:type=Memory","NonHeapMemoryUsage.used"] # 内存已用空间类加载
bash
# 加载总数:
java -jar /usr/local/tomcat/bin/cmdline-jmxclient-0.10.3.jar controlRole:tomcat 127.0.0.1:12345 java.lang:type=ClassLoading TotalLoadedClassCount
# 已加载:
java -jar /usr/local/tomcat/bin/cmdline-jmxclient-0.10.3.jar controlRole:tomcat 127.0.0.1:12345 java.lang:type=ClassLoading LoadedClassCount
# 已卸载:
java -jar /usr/local/tomcat/bin/cmdline-jmxclient-0.10.3.jar controlRole:tomcat 127.0.0.1:12345 java.lang:type=ClassLoading UnloadedClassCount
# Zabbix监控项的键值
jmx["java.lang:type=ClassLoading","TotalLoadedClassCount"] # 加载总数
jmx["java.lang:type=ClassLoading","LoadedClassCount"] # 已加载
jmx["java.lang:type=ClassLoading","UnloadedClassCount"] # 已卸载java线程
bash
# 总开启线程
java -jar /usr/local/tomcat/bin/cmdline-jmxclient-0.10.3.jar controlRole:tomcat 127.0.0.1:12345 java.lang:type=Threading TotalStartedThreadCount
# 活动线程
java -jar /usr/local/tomcat/bin/cmdline-jmxclient-0.10.3.jar controlRole:tomcat 127.0.0.1:12345 java.lang:type=Threading ThreadCount
# 线程峰值
java -jar /usr/local/tomcat/bin/cmdline-jmxclient-0.10.3.jar controlRole:tomcat 127.0.0.1:12345 java.lang:type=Threading PeakThreadCount
# Zabbix监控项的键值
jmx["java.lang:type=Threading","TotalStartedThreadCount"] # 总开启线程
jmx["java.lang:type=Threading","ThreadCount"] # 活动线程
jmx["java.lang:type=Threading","PeakThreadCount"] # 线程峰值tomcat线程
bash
# 最大线程
java -jar /usr/local/tomcat/bin/cmdline-jmxclient-0.10.3.jar - 127.0.0.1:12345 Catalina:name="http-nio-8080",type=ThreadPool maxThreads
# 当前线程
java -jar /usr/local/tomcat/bin/cmdline-jmxclient-0.10.3.jar - 127.0.0.1:12345 Catalina:name="http-nio-8080",type=ThreadPool currentThreadCount
# 繁忙线程
java -jar /usr/local/tomcat/bin/cmdline-jmxclient-0.10.3.jar - 127.0.0.1:12345 Catalina:name="http-nio-8080",type=ThreadPool currentThreadsBusy
# Zabbix监控项的键值
jmx["Catalina:type=ThreadPool,name=\"http-nio-8080\"",maxThreads] # 最大线程
jmx["Catalina:type=ThreadPool,name=\"http-nio-8080\"",currentThreadCount] # 当前线程
jmx["Catalina:type=ThreadPool,name=\"http-nio-8080\"",currentThreadsBusy] # 繁忙线程网络流量
bash
# 接收的字节
java -jar /usr/local/tomcat/bin/cmdline-jmxclient-0.10.3.jar - 127.0.0.1:12345 Catalina:name="http-nio-8080",type=GlobalRequestProcessor bytesReceived
# 发送的字节
java -jar /usr/local/tomcat/bin/cmdline-jmxclient-0.10.3.jar - 127.0.0.1:12345 Catalina:name="http-nio-8080",type=GlobalRequestProcessor bytesSent
# Zabbix监控项的键值
jmx["Catalina:type=GlobalRequestProcessor,name=\"http-nio-8080\"",bytesReceived] # 接收的字节
jmx["Catalina:type=GlobalRequestProcessor,name=\"http-nio-8080\"",bytesSent] # 发送的字节tomcat请求
bash
# tomcat请求数
java -jar /usr/local/tomcat/bin/cmdline-jmxclient-0.10.3.jar - 127.0.0.1:12345 Catalina:name="http-nio-8080",type=GlobalRequestProcessor requestCount
# tomcat出错请求
java -jar /usr/local/tomcat/bin/cmdline-jmxclient-0.10.3.jar - 127.0.0.1:12345 Catalina:name="http-nio-8080",type=GlobalRequestProcessor errorCount
# Zabbix监控项的键值
jmx["Catalina:type=GlobalRequestProcessor,name=\"http-nio-8080\"",requestCount] # tomcat请求数
jmx["Catalina:type=GlobalRequestProcessor,name=\"http-nio-8080\"",errorCount] # tomcat出错请求解决问题
bash
# 以上监控项找不到问题如:
org.archive.jmx.Client java.lang:name=PS Eden Space,type=MemoryPool is not a registered bean
# 可以大返回的匹配查看是否支持name字段,比如老版本个别的就PS开头
java -jar cmdline-jmxclient-0.10.3.jar - 127.0.0.1:12345 | grep MemoryPoolweb操作
添加主机
配置 —> 主机 —> 创建主机
- 主机名称:自定义
- 模板:
- Linux by Zabbix agent
- Generic Java JMX
- 群组:创建自定义
Tomcat集群群组 - Interfaces:
- 客户端:用于监控主机情况。端口10050
- JMX:用于监控Tomcat情况。端口12345
添加监控项
- 名称:监控Tomcat服务状态
- 类型:Zabbix客户端
- 键值:tomcat.status
- 信息类型:数字
- 更新间隔:5s
添加触发器
- 名称:Tomcat状态监测
- 严重性:严重
- 问题表现形式:
last(/tomcat-01/tomcat.status)=0 - 事件成功迭代:恢复表达式
- 恢复表达式:
last(/tomcat-01/tomcat.status)=1
添加图形
- 名称:Tomcat状态监控
- 监控项:选择刚添加的监控项名称
监控Tomcat服务状态
MySQL 监控
bash
cat > /etc/zabbix/zabbix_agent2.d/ << "EOF"
UserParameter=mysql.loguser,mysql -uroot -p"123123" -e "show full processlist"|wc -l
UserParameter=mysql.ping,mysqladmin -uroot -p"123123" -P3306 -h192.168.159.136 ping | grep -c alive
UserParameter=mysql.status[*],/opt/scripts/zabbix/chk_mysql.sh $1
EOF
mkdir -p /opt/scripts/zabbix
cat > /opt/scripts/zabbix/chk_mysql.sh << "EOF"
#!/bin/bash
# FileName: check_mysql.sh
# 用户名
MYSQL_USER='root'
# 密码
MYSQL_PWD='123123'
# 主机地址/IP
MYSQL_HOST='192.168.159.36'
# 端口
MYSQL_PORT='3306'
# 数据连接
MYSQL_CONN="/usr/bin/mysqladmin -u${MYSQL_USER} -p${MYSQL_PWD} -h${MYSQL_HOST} -P${MYSQL_PORT}"
# 参数是否正确
if [ $# -ne "1" ];then
echo "arg error!"
fi
# 获取数据
case $1 in
Uptime)
result=`${MYSQL_CONN} status | cut -f2 -d":" | cut -f1 -d"T"`
echo $result
;;
Com_update)
result=`${MYSQL_CONN} extended-status | grep -w "Com_update" | cut -d"|" -f3`
echo $result
;;
Slow_queries)
result=`${MYSQL_CONN} status | cut -f5 -d":" | cut -f1 -d"O"`
echo $result
;;
Com_select)
result=`${MYSQL_CONN} extended-status | grep -w "Com_select" | cut -d"|" -f3`
echo $result
;;
Com_rollback)
result=`${MYSQL_CONN} extended-status | grep -w "Com_rollback" | cut -d"|" -f3`
echo $result
;;
Questions)
result=`${MYSQL_CONN} status | cut -f4 -d":" | cut -f1 -d"S"`
echo $result
;;
Com_insert)
result=`${MYSQL_CONN} extended-status | grep -w "Com_insert" | cut -d"|" -f3`
echo $result
;;
Com_delete)
result=`${MYSQL_CONN} extended-status | grep -w "Com_delete" | cut -d"|" -f3`
echo $result
;;
Com_commit)
result=`${MYSQL_CONN} extended-status | grep -w "Com_commit" | cut -d"|" -f3`
echo $result
;;
Bytes_sent)
result=`${MYSQL_CONN} extended-status | grep -w "Bytes_sent" | cut -d"|" -f3`
echo $result
;;
Bytes_received)
result=`${MYSQL_CONN} extended-status | grep -w "Bytes_received" | cut -d"|" -f3`
echo $result
;;
Com_begin)
result=`${MYSQL_CONN} extended-status | grep -w "Com_begin" | cut -d"|" -f3`
echo $result
;;
*)
echo "Usage:$0(Uptime|Com_update|Slow_queries|Com_select|Com_rollback|Questions|Com_insert|Com_delete|Com_commit|Bytes_sent|Bytes_received|Com_begin)"
;;
esac
EOF
chmod +x /opt/scripts/zabbix/chk_mysql.sh
chown zabbix:zabbix /opt/scripts/zabbix/chk_mysql.sh
systemctl restart zabbix-agent2