Grafana + Prometheus로 16대 서버 통합 모니터링 — 설치부터 알림까지

# Prometheus 바이너리 설치

# 1. 다운로드 (최신 버전 확인: github.com/prometheus/prometheus/releases)
wget https://github.com/prometheus/prometheus/releases/download/v3.x.x/prometheus-3.x.x.linux-amd64.tar.gz
tar xvf prometheus-*.tar.gz

# 2. 바이너리 배치
sudo cp prometheus promtool /usr/local/bin/
sudo mkdir -p /etc/prometheus /var/lib/prometheus

# 3. 서비스 등록
sudo tee /etc/systemd/system/prometheus.service << 'EOF'
[Unit]
Description=Prometheus
After=network.target

[Service]
Type=simple
User=prometheus
ExecStart=/usr/local/bin/prometheus \
  --config.file=/etc/prometheus/prometheus.yml \
  --storage.tsdb.path=/var/lib/prometheus \
  --storage.tsdb.retention.time=30d \
  --web.listen-address=127.0.0.1:9090
Restart=always

[Install]
WantedBy=multi-user.target
EOF

sudo systemctl daemon-reload
sudo systemctl enable --now prometheus

# /etc/prometheus/prometheus.yml

global:
  scrape_interval: 15s      # 15초마다 메트릭 수집
  evaluation_interval: 15s  # 15초마다 알림 규칙 평가

rule_files:
  - "alert_rules.yml"       # 알림 규칙 파일

scrape_configs:
  # 시스템 메트릭 (16대 서버)
  - job_name: 'node'
    static_configs:
      - targets:
        - '<server-1>:9100'
        - '<server-2>:9100'
        - '<server-3>:9100'
        # ... 16대 전체 등록

    # IP → 서버명 변환 (대시보드 가독성)
    relabel_configs:
      - source_labels: [__address__]
        regex: '<server-1-ip>:.*'
        target_label: instance
        replacement: 'main-server'
      # ... 서버별 매핑

  # GPU 메트릭 (GPU 장착 서버만)
  - job_name: 'nvidia_gpu'
    scrape_interval: 5s      # GPU는 5초 간격
    static_configs:
      - targets: ['localhost:9835']
        labels:
          instance: 'gpu-server-1'
      - targets: ['<gpu-server-2>:9835']
        labels:
          instance: 'gpu-server-2'

  # 서비스 포트 감시
  - job_name: 'blackbox_tcp'
    metrics_path: /probe
    params:
      module: [tcp_connect]
    static_configs:
      - targets:
        - '<proxy-server>:80'   # HTTP
        - '<proxy-server>:443'  # HTTPS

# 바이너리 다운로드
wget https://github.com/prometheus/node_exporter/releases/download/v1.x.x/node_exporter-1.x.x.linux-amd64.tar.gz
tar xvf node_exporter-*.tar.gz
sudo cp node_exporter-*/node_exporter /usr/local/bin/

# systemd 서비스 등록
sudo tee /etc/systemd/system/node_exporter.service << 'EOF'
[Unit]
Description=Node Exporter
After=network.target

[Service]
Type=simple
User=node_exporter
ExecStart=/usr/local/bin/node_exporter \
  --collector.hwmon \
  --collector.textfile.directory=/var/lib/node_exporter/textfile_collector
Restart=always

[Install]
WantedBy=multi-user.target
EOF

sudo useradd -rs /bin/false node_exporter
sudo mkdir -p /var/lib/node_exporter/textfile_collector
sudo systemctl daemon-reload
sudo systemctl enable --now node_exporter

# nvidia_gpu_exporter 설치

# 바이너리 다운로드 (GitHub Releases)
wget https://github.com/utkuozdemir/nvidia_gpu_exporter/releases/download/v1.x.x/nvidia_gpu_exporter_1.x.x_linux_x86_64.tar.gz
tar xvf nvidia_gpu_exporter_*.tar.gz
sudo cp nvidia_gpu_exporter /usr/local/bin/

# systemd 등록 (기본 포트: 9835)
sudo tee /etc/systemd/system/nvidia_gpu_exporter.service << 'EOF'
[Unit]
Description=NVIDIA GPU Exporter
After=network.target

[Service]
Type=simple
ExecStart=/usr/local/bin/nvidia_gpu_exporter
Restart=always

[Install]
WantedBy=multi-user.target
EOF

sudo systemctl enable --now nvidia_gpu_exporter

# blackbox_exporter 설치

wget https://github.com/prometheus/blackbox_exporter/releases/download/v0.x.x/blackbox_exporter-0.x.x.linux-amd64.tar.gz
tar xvf blackbox_exporter-*.tar.gz
sudo cp blackbox_exporter-*/blackbox_exporter /usr/local/bin/

# blackbox.yml 설정
sudo tee /etc/prometheus/blackbox.yml << 'EOF'
modules:
  tcp_connect:
    prober: tcp
    timeout: 5s
EOF

# systemd 등록
sudo tee /etc/systemd/system/blackbox_exporter.service << 'EOF'
[Unit]
Description=Blackbox Exporter
After=network.target

[Service]
Type=simple
ExecStart=/usr/local/bin/blackbox_exporter \
  --config.file=/etc/prometheus/blackbox.yml
Restart=always

[Install]
WantedBy=multi-user.target
EOF

sudo systemctl enable --now blackbox_exporter

# Ubuntu/Debian
sudo apt install -y apt-transport-https software-properties-common
sudo mkdir -p /etc/apt/keyrings/
wget -q -O - https://apt.grafana.com/gpg.key | \
  gpg --dearmor | sudo tee /etc/apt/keyrings/grafana.gpg

echo "deb [signed-by=/etc/apt/keyrings/grafana.gpg] \
  https://apt.grafana.com stable main" | \
  sudo tee /etc/apt/sources.list.d/grafana.list

sudo apt update
sudo apt install grafana
sudo systemctl enable --now grafana-server

# /etc/grafana/provisioning/datasources/prometheus.yml

apiVersion: 1
datasources:
  - name: Prometheus
    type: prometheus
    access: proxy
    url: http://127.0.0.1:9090
    isDefault: true
    editable: false

# CPU 사용률 (%)
100 - (avg by(instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100)

# 메모리 사용률 (%)
(1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100

# 디스크 사용률 (%)
100 - (node_filesystem_avail_bytes{mountpoint="/"} / node_filesystem_size_bytes{mountpoint="/"} * 100)

# 서버 Uptime (일)
(time() - node_boot_time_seconds) / 86400

# GPU 온도 (°C)
nvidia_smi_temperature_gpu

# GPU VRAM 사용률 (%)
nvidia_smi_memory_used_bytes / nvidia_smi_memory_total_bytes * 100

# /etc/prometheus/alert_rules.yml

groups:
  # GPU 알림
  - name: gpu_alerts
    rules:
      - alert: GPUTemperatureWarning
        expr: nvidia_smi_temperature_gpu > 80
        for: 1m
        labels:
          severity: warning
        annotations:
          summary: "GPU 온도 경고 ({{ $value }}°C)"

      - alert: GPUTemperatureCritical
        expr: nvidia_smi_temperature_gpu > 85
        for: 30s
        labels:
          severity: critical
        annotations:
          summary: "GPU 온도 위험 ({{ $value }}°C)"

      - alert: GPUVRAMHigh
        expr: >
          nvidia_smi_memory_used_bytes
          / nvidia_smi_memory_total_bytes * 100 > 90
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "GPU VRAM 90% 초과"

  # 서버 알림
  - name: server_alerts
    rules:
      - alert: ServerDown
        expr: up{job="node"} == 0
        for: 5m
        labels:
          severity: critical
        annotations:
          summary: "서버 다운: {{ $labels.instance }}"

      - alert: DiskSpaceWarning
        expr: >
          100 - (node_filesystem_avail_bytes{mountpoint="/"}
          / node_filesystem_size_bytes{mountpoint="/"} * 100) > 90
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "디스크 90% 초과: {{ $labels.instance }}"

      - alert: HighCPU
        expr: >
          100 - (avg by(instance)
          (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 95
        for: 10m
        labels:
          severity: warning
        annotations:
          summary: "CPU 95% 초과: {{ $labels.instance }}"

      - alert: HighMemory
        expr: >
          (1 - node_memory_MemAvailable_bytes
          / node_memory_MemTotal_bytes) * 100 > 95
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "RAM 95% 초과: {{ $labels.instance }}"

  # 서비스 알림
  - name: service_alerts
    rules:
      - alert: ServiceDown
        expr: probe_success{job="blackbox_tcp"} == 0
        for: 2m
        labels:
          severity: critical
        annotations:
          summary: "서비스 다운: {{ $labels.instance }}"

# 점검 스크립트 핵심 구조

import requests
from datetime import datetime

PROMETHEUS_URL = "http://localhost:9090"

def query_prometheus(promql: str) -> dict:
    """Prometheus에 PromQL 쿼리를 실행합니다."""
    resp = requests.get(
        f"{PROMETHEUS_URL}/api/v1/query",
        params={"query": promql}
    )
    return resp.json()["data"]["result"]

def check_server_status():
    """전 서버 생존 상태를 확인합니다."""
    results = query_prometheus('up{job="node"}')
    for r in results:
        instance = r["metric"]["instance"]
        status = "정상" if r["value"][1] == "1" else "다운"
        print(f"  {instance}: {status}")

def check_cpu_temperature():
    """CPU 온도를 확인합니다."""
    results = query_prometheus(
        'node_hwmon_temp_celsius{chip=~".*coretemp.*|.*k10temp.*"}'
    )
    for r in results:
        instance = r["metric"]["instance"]
        temp = float(r["value"][1])
        status = "정상" if temp < 80 else "경고"
        print(f"  {instance}: {temp:.1f}°C [{status}]")

# ... 11개 점검 함수 실행 후 마크다운 로그 저장