Comprehensive Guide to Service Metrics Exporters#

1
#!/usr/bin/env python3
2
import psutil
3
import json
4
import os
5
import time
6
import argparse
7
from datetime import datetime
8

9
# Configuration
10
SERVICES = ["xdr-dashboard", "wazuh-indexer"]
11
METRICS_DIR = "metrics"
12

13
def ensure_metrics_dir():
14
    """Ensure metrics directory exists"""
15
    if not os.path.exists(METRICS_DIR):
16
        os.makedirs(METRICS_DIR)
17

18
def get_process_metrics(process_name):
19
    """Get metrics for a specific process"""
20
    metrics = {
21
        "cpu_percent": 0,
22
        "memory_percent": 0,
23
        "memory_mb": 0,
24
        "uptime_seconds": 0,
25
        "is_running": False
26
    }
27

28
    # Find all processes matching the name
29
    matching_processes = []
30
    for proc in psutil.process_iter(['pid', 'name', 'cmdline', 'create_time']):
31
        try:
32
            # Check if process name contains the service name
33
            if process_name in proc.info['name'] or any(process_name in cmd for cmd in proc.info.get('cmdline', []) if cmd):
34
                matching_processes.append(proc)
35
        except (psutil.NoSuchProcess, psutil.AccessDenied, psutil.ZombieProcess):
36
            pass
37

38
    if matching_processes:
39
        metrics["is_running"] = True
40

41
        # Aggregate metrics from all matching processes
42
        for proc in matching_processes:
43
            try:
44
                # Add CPU and memory usage
45
                process = psutil.Process(proc.info['pid'])
46
                metrics["cpu_percent"] += process.cpu_percent(interval=0.1)
47
                metrics["memory_percent"] += process.memory_percent()
48
                metrics["memory_mb"] += process.memory_info().rss / (1024 * 1024)  # Convert to MB
49

50
                # Calculate uptime
51
                current_time = time.time()
52
                process_start_time = proc.info['create_time']
53
                process_uptime = current_time - process_start_time
54
                metrics["uptime_seconds"] = max(metrics["uptime_seconds"], process_uptime)
55
            except (psutil.NoSuchProcess, psutil.AccessDenied):
56
                pass
57

58
    return metrics
59

60
def collect_metrics():
61
    """Collect metrics for configured services"""
62
    timestamp = datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
63
    all_metrics = {}
64

65
    for service in SERVICES:
66
        metrics = get_process_metrics(service)
67
        all_metrics[service] = metrics
68

69
        # Save individual service metrics
70
        service_filename = f"{METRICS_DIR}/{service}_{timestamp}.json"
71
        with open(service_filename, 'w') as f:
72
            json.dump(metrics, f, indent=2)
73

74
        # Print service metrics to console
75
        print(f"=== {service} Metrics ===")
76
        print(f"Running: {metrics['is_running']}")
77
        print(f"CPU Usage: {metrics['cpu_percent']:.2f}%")
78
        print(f"Memory Usage: {metrics['memory_percent']:.2f}% ({metrics['memory_mb']:.2f} MB)")
79
        print(f"Uptime: {metrics['uptime_seconds']/3600:.2f} hours")
80
        print("")
81

82
    # Save combined metrics
83
    combined_filename = f"{METRICS_DIR}/all_services_{timestamp}.json"
84
    with open(combined_filename, 'w') as f:
85
        json.dump(all_metrics, f, indent=2)
86

87
    return all_metrics
88

89
def main():
90
    parser = argparse.ArgumentParser(description='Service Metrics Exporter')
91
    parser.add_argument('--continuous', action='store_true', help='Run continuously')
92
    parser.add_argument('--interval', type=int, default=300, help='Interval in seconds (default: 300)')
93
    args = parser.parse_args()
94

95
    ensure_metrics_dir()
96

97
    if args.continuous:
98
        print(f"Starting continuous monitoring with {args.interval} second intervals...")
99
        try:
100
            while True:
101
                print(f"\n[{datetime.now()}] Collecting metrics...")
102
                collect_metrics()
103
                time.sleep(args.interval)
104
        except KeyboardInterrupt:
105
            print("Monitoring stopped by user.")
106
    else:
107
        collect_metrics()
108

109
if __name__ == "__main__":
110
    main()

1
#!/usr/bin/env python3
2
import requests
3
import re
4
import sys
5

6
def test_node_exporter():
7
    """Test if Node Exporter is working"""
8
    try:
9
        response = requests.get("http://localhost:9100/metrics", timeout=5)
10
        if response.status_code == 200:
11
            print("✅ Node Exporter is running and responding")
12
            return True
13
        else:
14
            print(f"❌ Node Exporter returned status code {response.status_code}")
15
            return False
16
    except requests.exceptions.RequestException as e:
17
        print(f"❌ Node Exporter is not accessible: {e}")
18
        return False
19

20
def test_process_exporter():
21
    """Test if Process Exporter is working"""
22
    try:
23
        response = requests.get("http://localhost:9256/metrics", timeout=5)
24
        if response.status_code == 200:
25
            print("✅ Process Exporter is running and responding")
26
            return True
27
        else:
28
            print(f"❌ Process Exporter returned status code {response.status_code}")
29
            return False
30
    except requests.exceptions.RequestException as e:
31
        print(f"❌ Process Exporter is not accessible: {e}")
32
        return False
33

34
def check_service_metrics(service_name):
35
    """Check if metrics for specific service are available"""
36
    try:
37
        response = requests.get("http://localhost:9256/metrics", timeout=5)
38
        if response.status_code == 200:
39
            if re.search(rf'process_name="{service_name}"', response.text):
40
                print(f"✅ Metrics for {service_name} found")
41
                return True
42
            else:
43
                print(f"❌ No metrics found for {service_name}")
44
                return False
45
        else:
46
            print(f"❌ Process Exporter returned status code {response.status_code}")
47
            return False
48
    except requests.exceptions.RequestException as e:
49
        print(f"❌ Process Exporter is not accessible: {e}")
50
        return False
51

52
if __name__ == "__main__":
53
    print("=== Metrics Exporter Validation ===")
54

55
    # Test basic connectivity
56
    node_ok = test_node_exporter()
57
    process_ok = test_process_exporter()
58

59
    # Check for specific services
60
    services = ["xdr-dashboard", "wazuh-indexer"]
61
    service_status = []
62

63
    if process_ok:
64
        for service in services:
65
            service_status.append(check_service_metrics(service))
66

67
    # Print summary
68
    print("\n=== Test Summary ===")
69
    print(f"Node Exporter: {'OK' if node_ok else 'FAIL'}")
70
    print(f"Process Exporter: {'OK' if process_ok else 'FAIL'}")
71

72
    if process_ok:
73
        for i, service in enumerate(services):
74
            print(f"Service {service}: {'OK' if service_status[i] else 'FAIL'}")
75

76
    # Determine overall status
77
    if all([node_ok, process_ok] + service_status):
78
        print("\n✅ All tests passed!")
79
        sys.exit(0)
80
    else:
81
        print("\n❌ Some tests failed!")
82
        sys.exit(1)

1
#!/bin/bash
2
# Set script variables
3
timestamp=$(date '+%Y-%m-%d %H:%M:%S')
4
node_exporter_port=9100
5
process_exporter_port=9256
6
services=("xdr-dashboard" "wazuh-indexer")
7

8
# ANSI color codes
9
GREEN='\033[0;32m'
10
RED='\033[0;31m'
11
NC='\033[0m' # No Color
12

13
echo "=== Metrics Health Check - $timestamp ==="
14

15
# Check if ports are open
16
echo -n "Checking Node Exporter port ($node_exporter_port): "
17
if nc -z localhost $node_exporter_port; then
18
    echo -e "${GREEN}OK${NC}"
19
else
20
    echo -e "${RED}FAIL${NC}"
21
fi
22

23
echo -n "Checking Process Exporter port ($process_exporter_port): "
24
if nc -z localhost $process_exporter_port; then
25
    echo -e "${GREEN}OK${NC}"
26
else
27
    echo -e "${RED}FAIL${NC}"
28
fi
29

30
# Check for metrics from each service
31
echo -e "\nChecking service metrics:"
32
for service in "${services[@]}"; do
33
    echo -n "  $service: "
34
    if curl -s "http://localhost:$process_exporter_port/metrics" | grep -q "process_name=\"$service\""; then
35
        echo -e "${GREEN}Found${NC}"
36
    else
37
        echo -e "${RED}Not found${NC}"
38
    fi
39
done
40

41
# Check system services
42
echo -e "\nChecking exporter services:"
43
echo -n "  node_exporter: "
44
if systemctl is-active --quiet node_exporter; then
45
    echo -e "${GREEN}Running${NC}"
46
else
47
    echo -e "${RED}Not running${NC}"
48
fi
49

50
echo -n "  process-exporter: "
51
if systemctl is-active --quiet process-exporter; then
52
    echo -e "${GREEN}Running${NC}"
53
else
54
    echo -e "${RED}Not running${NC}"
55
fi
56

57
# Add information about Prometheus scraping
58
echo -e "\nPrometheus targets can be checked at: http://localhost:9090/targets"

1
groups:
2
  - name: service_alerts
3
    rules:
4
      - alert: ServiceDown
5
        expr: process_up{name=~"xdr-dashboard|wazuh-indexer"} == 0
6
        for: 1m
7
        labels:
8
          severity: critical
9
        annotations:
10
          summary: "Service {{ $labels.name }} is down"
11
          description: "The service {{ $labels.name }} has been down for more than 1 minute."
12

13
      - alert: HighCPUUsage
14
        expr: rate(process_cpu_seconds_total{name=~"xdr-dashboard|wazuh-indexer"}[5m]) * 100 > 80
15
        for: 5m
16
        labels:
17
          severity: warning
18
        annotations:
19
          summary: "High CPU usage for {{ $labels.name }}"
20
          description: "{{ $labels.name }} is using more than 80% CPU for over 5 minutes."