Prometheus部署
官网查看下版本
https://www.prometheus.io/download/
github地址：
1、docker编排部署
部署编排文件
version: '3.3'
volumes:
prometheus_data: {}
grafana_data: {}
networks:
monitoring:
driver: bridge
services:
prometheus:
image: prom/prometheus:v2.45.2
container_name: prometheus
restart: always
volumes:
- /etc/localtime:/etc/localtime:ro
- /data/prometheus/:/etc/prometheus/
- prometheus_data:/prometheus
command:
- '--config.file=/etc/prometheus/prometheus.yml'
- '--storage.tsdb.path=/prometheus'
- '--web.console.libraries=/usr/share/prometheus/console_libraries'
- '--web.console.templates=/usr/share/prometheus/consoles'
#热加载配置
- '--web.enable-lifecycle'
#api配置
#- '--web.enable-admin-api'
#历史数据最大保留时间，默认15天
- '--storage.tsdb.retention.time=30d'
networks:
- monitoring
links:
- alertmanager
#- cadvisor
- node_exporter
expose:
- '9090'
ports:
- 9090:9090
#depends_on:
#- cadvisor
alertmanager:
image: prom/alertmanager:v0.26.0
container_name: alertmanager
restart: always
volumes:
- /etc/localtime:/etc/localtime:ro
- /data/alertmanager/:/etc/alertmanager/
command:
- '--config.file=/etc/alertmanager/config.yml'
- '--storage.path=/alertmanager'
networks:
- monitoring
expose:
- '9093'
ports:
- 9093:9093
node_exporter:
image: prom/node-exporter:v1.7.0
hostname: IT-Prometheus
container_name: node-exporter
restart: always
volumes:
- /etc/localtime:/etc/localtime:ro
- /proc:/host/proc:ro
- /sys:/host/sys:ro
- /:/rootfs:ro
command:
- '--path.procfs=/host/proc'
- '--path.sysfs=/host/sys'
- '--collector.filesystem.ignored-mount-points=^/(sys|proc|dev|host|etc|rootfs/var/lib/docker)($$|/)'
networks:
- monitoring
expose:
- '9100'
grafana:
image: grafana/grafana:10.2.0
container_name: grafana
restart: always
volumes:
- /etc/localtime:/etc/localtime:ro
- grafana_data:/var/lib/grafana
- /data/grafana/provisioning/:/etc/grafana/provisioning/
env_file:
- ./grafana/config.monitoring
networks:
- monitoring
links:
- prometheus
ports:
- 3000:3000
depends_on:
- prometheus
2、alertmanager配置文件
目录下config.yml
global:
#邮箱服务器
smtp_smarthost: 'smtp-mail.qq.com:587'
#发邮件的邮箱
smtp_from: 'xx@qq.com'
#发邮件的邮箱用户名，也就是你的邮箱
smtp_auth_username: 'xx@qq.com'
#发邮件的邮箱密码
smtp_auth_password: '123456'
#进行tls验证
smtp_require_tls: true
route:
group_by: ['alertname','instance','job']
# 当收到告警的时候，等待group_wait配置的时间，看是否还有告警，如果有就一起发出去
group_wait: 10s
# 如果上次告警信息发送成功，此时又来了一个新的告警数据，则需要等待group_interval配置的时间才可以发送出去
group_interval: 5m
# 如果上次告警信息发送成功，且问题没有解决，则等待 repeat_interval配置的时间再次发送告警数据
repeat_interval: 12h
# 全局报警组，这个参数是必选的
receiver: email
routes:
- receiver: email
match_re:
#公司组UP
company: UP
- receiver: email_cs
match_re:
#公司组cs
company: CS
templates:
#邮件模板文件
- './template/emails.tmpl'
receivers:
- name: 'email'
#收邮件的邮箱
email_configs:
- to: 'x1@qq.com'
send_resolved: true
headers: { Subject: " 【服务器告警】 {{ .CommonLabels.alertname }} " }
html: '{{ template "moban.html" . }}'
- to: 'x2@qq.com'
send_resolved: true
headers: { Subject: " 【服务器告警】 {{ .CommonLabels.alertname }} " }
html: '{{ template "moban.html" . }}'
- name: 'email_cs'
#测试收邮件的邮箱
email_configs:
- to: 'x1@qq.com'
send_resolved: true
headers: { Subject: " [CS]{{ .CommonLabels.instance }} {{ .CommonLabels.alertname }} " }
html: '{{ template "moban.html" . }}'
inhibit_rules:
- source_match:
severity: 'critical'
target_match:
severity: 'warning'
equal: ['alertname', 'dev', 'instance']
template目录下邮件模板emails.tmpl
{{ define "__alertmanager" }}Alertmanager{{ end }}
{{ define "__subject" }}[{{ .Status | toUpper }}{{ if eq .Status "firing" }}:{{ .Alerts.Firing | len }}{{ end }}] {{ .GroupLabels.SortedPairs.Values | join " " }} {{ if gt (len .CommonLabels) (len .GroupLabels) }}({{ with .CommonLabels.Remove .GroupLabels.Names }}{{ .Values | join " " }}{{ end }}){{ end }}{{ end }}
{{ define "__description" }}{{ end }}
{{ define "moban.html" }}
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
<html xmlns="http://www.w3.org/1999/xhtml" xmlns="http://www.w3.org/1999/xhtml" style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;">
<head style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;">
<meta name="viewport" content="width=device-width" style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;" />
<meta http-equiv="Content-Type" content="text/html; charset=UTF-8" style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;" />
<title style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;">{{ template "__subject" . }}</title>
<style>
@media only screen and (max-width: 640px) {
body {
padding: 0 !important;
}
h1,
h2,
h3,
h4 {
font-weight: 800 !important;
margin: 20px 0 5px !important;
}
h1 {
font-size: 22px !important;
}
h2 {
font-size: 18px !important;
}
h3 {
font-size: 16px !important;
}
.container {
padding: 0 !important;
width: 100% !important;
}
.content {
padding: 0 !important;
}
.content-wrap {
padding: 10px !important;
}
.invoice {
width: 100% !important;
}
}
</style>
</head>
<body itemscope itemtype="https://schema.org/EmailMessage" style="margin: 0; font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; -webkit-font-smoothing: antialiased; -webkit-text-size-adjust: none; height: 100%; line-height: 1.6em; background-color: #f6f6f6; width: 100%;">
<table class="body-wrap" style="margin: 0; font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; background-color: #f6f6f6; width: 100%;" width="100%" bgcolor="#f6f6f6">
<tr style="margin: 0; font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px;">
<td style="margin: 0; font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top;" valign="top"></td>
<td class="container" width="600" style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; display: block; max-width: 600px; margin: 0 auto; clear: both;" valign="top">
<div class="content" style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; max-width: 600px; margin: 0 auto; display: block; padding: 20px;">
<table class="main" width="100%" cellpadding="0" cellspacing="0" style="margin: 0; font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; background-color: #fff; border: 1px solid #e9e9e9; border-radius: 3px;" bgcolor="#fff">
<tr style="margin: 0; font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px;">
{{ if gt (len .Alerts.Firing) 0 }}
<td class="alert alert-warning" style="margin: 0; font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; vertical-align: top; font-size: 16px; color: #fff; font-weight: 500; padding: 0 20px 20px 20px; text-align: center; border-radius: 3px 3px 0 0; background-color: #E6522C;" valign="top" align="center" bgcolor="#E6522C">
{{ range .GroupLabels.SortedPairs }}
<br>{{ .Value }}
{{ end }} 告警 ！！请尽快处理
</td>
{{ else }}
<td class="alert alert-good" style="margin: 0; font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; vertical-align: top; font-size: 16px; color: #fff; font-weight: 500; padding: 0 20px 20px 20px; text-align: center; border-radius: 3px 3px 0 0; background-color: #68B90F;" valign="top" align="center" bgcolor="#68B90F">
{{ range .GroupLabels.SortedPairs }}
<br>{{ .Value }}
{{ end }} 告警 ！！已解决
</td>
{{ end }}
</tr>
<tr style="margin: 0; font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px;">
<td class="content-wrap" style="margin: 0; font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; padding: 30px;" valign="top">
<table width="100%" cellpadding="0" cellspacing="0" style="margin: 0; font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px;">
<tr style="margin: 0; font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px;">
<td class="content-block" style="margin: 0; font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; padding: 0 0 20px;" valign="top">
<a href="http://10.201.216.85:3000/d/Kdh0OoSGz/" class="btn-primary" style="margin: 0; font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; text-decoration: none; color: #FFF; background-color: #348eda; border: solid #348eda; border-width: 10px 20px; line-height: 2em; font-weight: bold; text-align: center; cursor: pointer; display: inline-block; border-radius: 5px; text-transform: capitalize;">查看详细</a>
</td>
</tr>
{{ if gt (len .Alerts.Firing) 0 }}
<tr style="margin: 0; font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px;">
<td class="content-block" style="margin: 0; font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; padding: 0 0 20px;" valign="top">
<strong style="margin: 0; font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px;">[{{ .Alerts.Firing | len }}] 告警</strong>
</td>
</tr>
{{ end }}
{{ range .Alerts.Firing }}
<tr style="margin: 0; font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px;">
<td class="content-block" style="margin: 0; font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; padding: 0 0 20px;" valign="top">
<strong style="margin: 0; font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px;">Labels</strong><br style="margin: 0; font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px;">
{{ range .Labels.SortedPairs }}{{ .Name }} = {{ .Value }}<br style="margin: 0; font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px;">{{ end }}
<br>{{ if gt (len .Annotations) 0 }}<strong style="margin: 0; font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px;">标注：</strong><br style="margin: 0; font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px;">{{ end }}
{{ range .Annotations.SortedPairs }}{{ .Name }} = {{ .Value }}<br style="margin: 0; font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px;">{{ end }}
<br><a href="{{ .GeneratorURL }}" style="margin: 0; font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; color: #348eda; text-decoration: underline;">Source</a><br style="margin: 0; font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px;">
</td>
</tr>
{{ end }}
{{ if gt (len .Alerts.Resolved) 0 }}
{{ if gt (len .Alerts.Firing) 0 }}
<tr style="margin: 0; font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px;">
<td class="content-block" style="margin: 0; font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; padding: 0 0 20px;" valign="top">
<br style="margin: 0; font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px;">
<hr style="margin: 0; font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px;">
<br style="margin: 0; font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px;">
</td>
</tr>
{{ end }}
<tr style="margin: 0; font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px;">
<td class="content-block" style="margin: 0; font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; padding: 0 0 20px;" valign="top">
<strong style="margin: 0; font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px;">[{{ .Alerts.Resolved | len }}] 已解决</strong>
</td>
</tr>
{{ end }}
{{ range .Alerts.Resolved }}
<tr style="margin: 0; font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px;">
<td class="content-block" style="margin: 0; font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; padding: 0 0 20px;" valign="top">
<strong style="margin: 0; font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px;">Labels</strong><br style="margin: 0; font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px;">
{{ range .Labels.SortedPairs }}{{ .Name }} = {{ .Value }}<br style="margin: 0; font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px;">{{ end }}
<br>{{ if gt (len .Annotations) 0 }}<strong style="margin: 0; font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px;">标注：</strong><br style="margin: 0; font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px;">{{ end }}
{{ range .Annotations.SortedPairs }}{{ .Name }} = {{ .Value }}<br style="margin: 0; font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px;">{{ end }}
<br><a href="{{ .GeneratorURL }}" style="margin: 0; font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; color: #348eda; text-decoration: underline;">Source</a><br style="margin: 0; font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px;">
</td>
</tr>
{{ end }}
</table>
</td>
</tr>
</table>
<div class="footer" style="margin: 0; font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; width: 100%; clear: both; color: #999; padding: 20px;">
<table width="100%" style="margin: 0; font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px;">
<tr style="margin: 0; font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px;">
<td class="aligncenter content-block" style="margin: 0; font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; vertical-align: top; padding: 0 0 20px; text-align: center; color: #999; font-size: 12px;" valign="top" align="center"><a href="http://10.201.216.59/" style="margin: 0; font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; text-decoration: underline; color: #999; font-size: 12px;">Sent by Quectel 常州IT部 Computer Center</a></td>
</tr>
</table>
</div></div>
</td>
<td style="margin: 0; font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top;" valign="top"></td>
</tr>
</table>
</body>
</html>
{{ end }}
3、prometheus配置文件
目录下prometheus.yml
# 全局配置
global:
scrape_interval: 15s # 将搜刮间隔设置为每15秒一次。默认是每1分钟一次。
evaluation_interval: 15s # 每15秒评估一次规则。默认是每1分钟一次。
# Alertmanager 配置
alerting:
alertmanagers:
- static_configs:
- targets: ['alertmanager:9093']
# 报警(触发器)配置
rule_files:
- "alert.yml"
# 搜刮配置
scrape_configs:
- job_name: 'prometheus'
# 覆盖全局默认值，每15秒从该作业中刮取一次目标
scrape_interval: 15s
static_configs:
- targets: ['localhost:9090']
- job_name: 'alertmanager'
scrape_interval: 15s
static_configs:
- targets: ['alertmanager:9093']
- job_name: 'linux-Server'
scrape_interval: 15s
static_configs:
- targets: ['node_exporter:9100']
labels:
instance: 192.168.0.2
hosts: IT-Prometheus
teams: IT
company: UP
- targets: ['10.1.6.10:180']
labels:
instance: 10.1.6.10
hosts: IT-Nginx
teams: IT
company: UP
- job_name: 'windows-Server'
scrape_interval: 15s
static_configs:
# 宿主机Hyperv
- targets: ['10.1.16.18:180']
labels:
instance: 10.1.16.18
hosts: IT-Hyperv-01
teams: IT
company: cs
- targets: ['10.11.6.18:180']
labels:
instance: 10.11.6.18
hosts: IT-HYPERV-02
teams: IT
company: UP
目录下alert.yml
groups:
- name: 主机状态-监控告警
rules:
# 对任何实例超过30秒无法联系的情况发出警报
- alert: 服务告警
expr: up == 0
for: 30s
labels:
severity: critical
annotations:
summary: "服务异常,实例:{{ $labels.instance }}({{ $labels.hosts }})"
description: "{{ $labels.job }} 服务已关闭"
- alert: Windows服务器CPU使用率过高告警
expr: (1 - avg by (instance,hosts,teams,company) (irate(windows_cpu_time_total{job="windows-Server",mode="idle"}[10m]))) * 100 > 90
for: 30s
labels:
severity: critical
#company: CS
annotations:
summary: "{{ $labels.instance }}({{ $labels.hosts }}) CPU使用率过高！"
description: "{{ $labels.instance }} CPU使用大于90%(目前使用:{{humanize $value}}%)"
- alert: Windows服务器内存使用率过高告警
expr: 100-100*windows_os_physical_memory_free_bytes{job="windows-Server"} / windows_cs_physical_memory_bytes{job="windows-Server"} > 90
for: 30s
labels:
severity: critical
#company: CS
annotations:
summary: "{{ $labels.instance }}({{ $labels.hosts }}) 内存使用率过高！"
description: "{{ $labels.instance }} 内存使用大于90%(目前使用:{{humanize $value}}%)"
- alert: Windows服务器磁盘分区使用率过高告警
expr: (max by(instance,hosts,teams,company)(1-windows_logical_disk_free_bytes{job=~"windows-Server",volume!~"HarddiskVolume.*"}/windows_logical_disk_size_bytes{job=~"windows-Server",volume!~"HarddiskVolume.*"})*100) > 90
for: 90s
labels:
severity: critical
#company: CS
annotations:
summary: "{{ $labels.instance }}({{ $labels.hosts }}) 磁盘分区使用率过高！"
description: "{{ $labels.instance }} 磁盘分区使用大于90%(目前最大分区磁盘使用:{{humanize $value}}%)"
- alert: Windows服务器IO写入超过阈值告警
expr: max by (instance,hosts,teams,company) (irate(windows_logical_disk_writes_total[30m])) > 250
for: 30s
labels:
severity: critical
company: CS
annotations:
summary: "{{ $labels.instance }}({{ $labels.hosts }}) IO写入超过阈值！"
description: "{{ $labels.instance }} 设定30秒内IO写入阈值大于250(目前IO写入:{{humanize $value}})"
- alert: Windows服务器IO被读取超过阈值告警
expr: -max by (instance,hosts,teams,company) (irate(windows_logical_disk_reads_total[30m])) < -180
for: 30s
labels:
severity: critical
company: CS
annotations:
summary: "{{ $labels.instance }}({{ $labels.hosts }}) IO被读取超过阈值！"
description: "{{ $labels.instance }} 设定30秒内IO被读取阈值小于-180(目前IO读取:{{humanize $value}})"
4、grafana配置文件
目录下config.monitoring设置初始配置密码
GF_SECURITY_ADMIN_PASSWORD=password
GF_USERS_ALLOW_SIGN_UP=false