# @see: https://hub.docker.com/r/prom/prometheus/tags
docker pull docker.io/prom/prometheus:v2.30.3

sudo mkdir -p /etc/prometheus

# ----- prometheus.yml -----
sudo cat <<-'EOF'>/etc/prometheus/prometheus.yml
# my global config
global:
  scrape_interval: 15s # Set the scrape interval to every 15 seconds. Default is every 1 minute.
  evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute.
  # scrape_timeout is set to the global default (10s).

# Alertmanager configuration
alerting:
  alertmanagers:
    - static_configs:
        - targets:
           - 10.111.178.63:9093

# Load rules once and periodically evaluate them according to the global 'evaluation_interval'.
rule_files:
  - "emqx-alert-rules.yml"
  - "haproxy-alert-rules.yml"
  - "jvm-alert-rules.yml"
  - "kafka-alert-rules.yml"
  - "node-alert-rules.yml"
  - "prometheus-alert-rules.yml"
  - "redis-alert-rules.yml"

# A scrape configuration containing exactly one endpoint to scrape:
# Here it's Prometheus itself.
scrape_configs:
  # The job name is added as a label job=<job_name> to any timeseries scraped from this config.
  - job_name: "prometheus"
    # metrics_path defaults to '/metrics'
    # scheme defaults to 'http'.
    static_configs:
      - targets: ["localhost:9090"]
  - job_name: node
    static_configs:
    - targets: ['10.111.0.111:9100']
      labels:
        instance: collect-node1
    - targets: ['10.111.0.112:9100']
      labels:
        instance: collect-node2
    - targets: ['10.111.178.79:9100']
      labels:
        instance: web-node2
    - targets: ['10.111.178.78:9100']
      labels:
        instance: web-node3
    - targets: ['10.111.178.60:9100']
      labels:
        instance: compute-node1
    - targets: ['10.111.0.11:9100']
      labels:
        instance: kafka-follower1
    - targets: ['10.111.0.12:9100']
      labels:
        instance: kafka-follower2
    - targets: ['10.29.69.150:9100']
      labels:
        instance: kafka-leader1
    - targets: ['10.111.178.58:9100']
      labels:
        instance: sink-node1
    - targets: ['10.111.178.57:9100']
      labels:
        instance: sink-node2
    - targets: ['10.111.178.70:9100']
      labels:
        instance: repo-node1
    - targets: ['10.111.178.63:9100']
      labels:
        instance: devops-node1
    - targets: ['10.111.178.72:9100']
      labels:
        instance: emr-header-1.cluster-125585
    - targets: ['10.111.178.73:9100']
      labels:
        instance: emr-header-2.cluster-125585
    - targets: ['10.111.178.71:9100']
      labels:
        instance: emr-worker-1.cluster-125585
    - targets: ['10.111.178.74:9100']
      labels:
        instance: emr-worker-2.cluster-125585
    - targets: ['10.111.178.75:9100']
      labels:
        instance: emr-worker-3.cluster-125585
    - targets: ['10.111.178.76:9100']
      labels:
        instance: emr-worker-4.cluster-125585
  - job_name: haproxy
    static_configs:
    - targets: ['10.111.0.111:9101']
      labels:
        instance: collect-node1:9101
    - targets: ['10.111.0.112:9101']
      labels:
        instance: collect-node2:9101
  - job_name: iam-gateway
    metrics_path: /actuator/prometheus
    static_configs:
    - targets: ['10.111.178.79:18086']
      labels:
        instance: web-node2:18086
    - targets: ['10.111.178.78:18086']
      labels:
        instance: web-node3:18086
  - job_name: kafka
    static_configs:
    - targets: ['10.111.0.11:10105']
      labels:
        instance: kafka-follower1:10105
    - targets: ['10.111.0.12:10105']
      labels:
        instance: kafka-follower2:10105
    - targets: ['10.29.69.150:10105']
      labels:
        instance: kafka-leader1:10105
  - job_name: emq
    static_configs:
    - targets: ['10.111.0.111:9091']
      labels:
        instance: collect-node1:9091
    - targets: ['10.111.0.112:9091']
      labels:
        instance: collect-node2:9091
EOF

# see: https://hub.docker.com/r/grafana/grafana/tags
docker pull docker.io/grafana/grafana:8.2.2

sudo mkdir -p /mnt/disk1/grafana
sudo chmod -R 777 /mnt/disk1/grafana
docker run -d --name=grafana1 -p 3000:3000 --network host --restart=always -v /mnt/disk1/grafana:/var/lib/grafana docker.io/grafana/grafana:8.2.2

# ------ Downloading installation -------
sudo cd /tmp
sudo curl -OL https://github.91chifun.workers.dev/https://github.com//prometheus/node_exporter/releases/download/v1.2.2/node_exporter-1.2.2.linux-amd64.tar.gz
sudo tar -xf node_exporter*
sudo cp node_exporter*/node_exporter /usr/sbin/

# ------ Make configuration -------
sudo cat <<-'EOF' >/etc/sysconfig/node_exporter
OPTIONS="--collector.systemd \
--collector.mdadm \
--collector.tcpstat \
--collector.processes \
--web.listen-address=:9100 \
--web.telemetry-path=/metrics"
EOF

sudo cat <<-'EOF' >/etc/systemd/system/node_exporter.service
[Unit]
Description=Node Exporter

[Service]
User=root
Group=root
EnvironmentFile=-/etc/sysconfig/node_exporter
ExecStart=/usr/sbin/node_exporter $OPTIONS
ExecReload=/bin/kill -s HUP
StandardOutput=journal
StandardError=journal
LimitNOFILE=64
LimitNPROC=64
LimitCORE=infinity
TimeoutStartSec=10
TimeoutSec=300
Restart=always

[Install]
WantedBy=multi-user.target
EOF

sudo systemctl daemon-reload
sudo systemctl enable node_exporter
sudo systemctl restart node_exporter
sudo systemctl status node_exporter
sudo journalctl -u node_exporter -f

# 测试访问
curl http://localhost:9100/metrics

sudo cat <<-'EOF' >/etc/init.d/node_exporter.sh
#!/bin/bash
#/*
# * Copyright 2017 ~ 2025 the original author or authors. 
# *
# * Licensed under the Apache License, Version 2.0 (the "License");
# * you may not use this file except in compliance with the License.
# * You may obtain a copy of the License at
# *
# *      http://www.apache.org/licenses/LICENSE-2.0
# *
# * Unless required by applicable law or agreed to in writing, software
# * distributed under the License is distributed on an "AS IS" BASIS,
# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# * See the License for the specific language governing permissions and
# * limitations under the License.
# */

[ -f /etc/sysconfig/network ] && . /etc/sysconfig/network
[ "$NETWORKING" = "no" ] && exit 0

# Environment definition.
binFile="$(command -v node_exporter)"
if [ -f /etc/sysconfig/node_exporter ]; then
  . /etc/sysconfig/node_exporter
  export NODE_EXPORTER_OPTIONS=$OPTIONS
fi

function start() {
  local pids=$(getPids)
  if [ -z "$pids" ]; then
    nohup $binFile $NODE_EXPORTER_OPTIONS > "/var/log/node_exporter.stdout" 2>&1 &
    echo -n "Starting node exporter ..."
    while true
    do
      pids=$(getPids)
      if [ "$pids" == "" ]; then
        echo -n ".";
        sleep 0.8;
      else
        echo $pids >"/run/node_exporter.pid"
        break
      fi
    done
    echo -e "\nStarted node_exporter on "$pids
  else
    echo "Node exporter process is running "$pids
  fi
}

function stop() {
  local pids=$(getPids)
  if [ -z "$pids" ]; then
    echo "Node exporter not running!"
  else
    echo -n "Stopping node_exporter for $pids ..."
    kill -s TERM $pids
    while true
    do
      pids=$(getPids)
      if [ "$pids" == "" ]; then
        \rm -f /run/node_exporter.pid
        break
      else
        echo -n ".";
        sleep 0.8;
      fi
    done
    echo -e "\nStopped node_exporter !"
  fi
}

function status() {
  ps -ef | grep -v grep | grep $binFile
}

function getPids() {
  local pids=$(ps ax | grep -i "$binFile" | grep -v grep | awk '{print $1}')
  echo $pids # Output execution result value.
  return 0 # Return the execution result code.
}

# --- Main call. ---
CMD=$1
case $CMD in
  status)
    status
    ;;
  start)
    start
    ;;
  stop)
    stop
    ;;
  restart)
    stop
    start
    ;;
    *)
  echo $"Usage: {start|stop|restart|status}"
  exit 2
esac
EOF

chmod +x /etc/init.d/node_exporter.sh
/etc/init.d/node_exporter.sh restart
/etc/init.d/node_exporter.sh status

# 测试访问
curl http://localhost:9100/metrics

sudo mkdir -p /etc/alertmanager

sudo cat <<-'EOF'>/etc/alertmanager/alertmanager.yml
global:
  resolve_timeout: 5m
  smtp_smarthost: 'smtp.163.com:465'
  smtp_from: 'you_email_user@163.com'
  smtp_auth_username: 'you_email_user@163.com'
  smtp_auth_password: 'you_email_password'
  smtp_require_tls: false # see: smtp 454 Command not permitted when TLS active
  smtp_hello: '163.com'
  #wechat_api_url: 'https://qyapi.weixin.qq.com/cgi-bin/'

templates:
  - 'template/*.tmpl'

route:
  group_by: ['alertname'] # 报警分组依据
  group_wait: 10s # 最初即第一次等待多久时间发送一组警报的通知
  group_interval: 10s # 在发送新警报前的等待时间
  repeat_interval: 1m # 发送重复警报的周期, 对于email此项不可以设置过低, 以防smtp服务器拒绝
  receiver: 'email' # 对应以下receivers

receivers:
  - name: 'email'
    email_configs:
      - to: '983708408@qq.com'
        html: '{{ template "email.tmpl" . }}'
        headers: { Subject: "[WARN] 报警邮件"}
  #- name: 'webhook'
  #  webhook_configs:
  #    - url: 'http://127.0.0.1:5001'
  #      send_resolved: true
  #- name: 'wechat'
  #  wechat_configs:
  #    - send_resolved: true
  #      to_party: '1' # 接收组ID
  #      agent_id: '1000002' # 企业微信->自定应用->AgentId
  #      corp_id: '******' # 企业信息(我的企业->CorpId[在底部])
  #      api_secret: '******' # 企业微信(企业微信->自定应用->Secret)
  #      message: '{{ template "test_wechat.html" . }}'

# 抑制器配置
inhibit_rules:
  - source_match: # 源标签警报触发时抑制含有目标标签的警报，在当前警报匹配 status: 'High'
      severity: 'critical'
      status: 'High'
    target_match:
      severity: 'warning'
    equal: ['alertname', 'dev', 'instance']
EOF