Monitoring Linux Resource Usage with Wazuh#

1
<!-- CPU, memory, disk metric -->
2
<localfile>
3
  <log_format>full_command</log_format>
4
  <command>echo $(top -bn1 | grep Cpu | awk '{print $2+$4+$6+$12+$14+$16}' ; free -m | awk 'NR==2{printf "%.2f\t\t\n", $3*100/$2 }' ; df -h | awk '$NF=="/"{print $5}'|sed 's/%//g')</command>
5
  <alias>general_health_metrics</alias>
6
  <out_format>$(timestamp) $(hostname) general_health_check: $(log)</out_format>
7
  <frequency>30</frequency>
8
</localfile>
9

10
<!-- load average metrics -->
11
<localfile>
12
  <log_format>full_command</log_format>
13
  <command>uptime | grep load | awk '{print $(NF-2),$(NF-1),$NF}' | sed 's/\,\([0-9]\{1,2\}\)/.\1/g'</command>
14
  <alias>load_average_metrics</alias>
15
  <out_format>$(timestamp) $(hostname) load_average_check: $(log)</out_format>
16
  <frequency>30</frequency>
17
</localfile>
18

19
<!-- memory metrics -->
20
<localfile>
21
  <log_format>full_command</log_format>
22
  <command>free --bytes| awk 'NR==2{print $3,$7}'</command>
23
  <alias>memory_metrics</alias>
24
  <out_format>$(timestamp) $(hostname) memory_check: $(log)</out_format>
25
  <frequency>30</frequency>
26
</localfile>
27

28
<!-- disk metrics -->
29
<localfile>
30
  <log_format>full_command</log_format>
31
  <command>df -B1 | awk '$NF=="/"{print $3,$4}'</command>
32
  <alias>disk_metrics</alias>
33
  <out_format>$(timestamp) $(hostname) disk_check: $(log)</out_format>
34
  <frequency>30</frequency>
35
</localfile>

1
<!-- Monitor specific process CPU usage -->
2
<localfile>
3
  <log_format>full_command</log_format>
4
  <command>ps aux | grep -E 'nginx|apache2' | awk '{sum+=$3} END {print sum}'</command>
5
  <alias>webserver_cpu</alias>
6
  <out_format>$(timestamp) $(hostname) webserver_cpu_usage: $(log)</out_format>
7
  <frequency>60</frequency>
8
</localfile>
9

10
<!-- Monitor Docker containers -->
11
<localfile>
12
  <log_format>full_command</log_format>
13
  <command>docker stats --no-stream --format "{{.Container}}:{{.CPUPerc}}:{{.MemPerc}}" 2>/dev/null || echo "No Docker"</command>
14
  <alias>docker_stats</alias>
15
  <out_format>$(timestamp) $(hostname) docker_stats: $(log)</out_format>
16
  <frequency>60</frequency>
17
</localfile>

1
<!-- Network interface statistics -->
2
<localfile>
3
  <log_format>full_command</log_format>
4
  <command>cat /proc/net/dev | grep -E 'eth0|ens' | awk '{print $1,$2,$10}'</command>
5
  <alias>network_stats</alias>
6
  <out_format>$(timestamp) $(hostname) network_stats: $(log)</out_format>
7
  <frequency>30</frequency>
8
</localfile>
9

10
<!-- Active connections -->
11
<localfile>
12
  <log_format>full_command</log_format>
13
  <command>ss -s | grep -E 'TCP:|UDP:' | awk '{print $1,$2}'</command>
14
  <alias>connection_stats</alias>
15
  <out_format>$(timestamp) $(hostname) connection_stats: $(log)</out_format>
16
  <frequency>60</frequency>
17
</localfile>

1
<!-- Sustained high CPU usage -->
2
<rule id="100061" level="14" frequency="5" timeframe="300">
3
  <if_sid>100056</if_sid>
4
  <description>Sustained high CPU usage detected</description>
5
  <options>alert_by_email</options>
6
</rule>
7

8
<!-- Rapid disk usage increase -->
9
<rule id="100062" level="12" frequency="3" timeframe="300">
10
  <if_sid>100057</if_sid>
11
  <description>Rapid disk usage increase - possible log flooding</description>
12
</rule>
13

14
<!-- Memory leak detection -->
15
<rule id="100063" level="10">
16
  <if_sid>100059</if_sid>
17
  <match>memory_available_bytes: 0</match>
18
  <description>System out of memory</description>
19
</rule>
20

21
<!-- High load with low CPU (I/O bottleneck) -->
22
<rule id="100064" level="10">
23
  <if_sid>100058</if_sid>
24
  <regex>^([5-9]\.|[1-9]\d+\.)</regex>
25
  <description>High system load detected: $(1min_loadAverage)</description>
26
</rule>

1
{
2
  "version": "7.14.2",
3
  "objects": [{
4
    "id": "linux-performance-dashboard",
5
    "type": "dashboard",
6
    "attributes": {
7
      "title": "Linux Performance Monitoring",
8
      "hits": 0,
9
      "description": "Comprehensive Linux resource monitoring dashboard",
10
      "panelsJSON": "[{\"gridData\":{\"x\":0,\"y\":0,\"w\":24,\"h\":15},\"panelIndex\":\"1\",\"embeddableConfig\":{\"title\":\"CPU Usage Over Time\"},\"panelRefName\":\"panel_cpu_usage\"},{\"gridData\":{\"x\":24,\"y\":0,\"w\":24,\"h\":15},\"panelIndex\":\"2\",\"embeddableConfig\":{\"title\":\"Memory Usage Over Time\"},\"panelRefName\":\"panel_memory_usage\"},{\"gridData\":{\"x\":0,\"y\":15,\"w\":24,\"h\":15},\"panelIndex\":\"3\",\"embeddableConfig\":{\"title\":\"Load Average Trends\"},\"panelRefName\":\"panel_load_average\"},{\"gridData\":{\"x\":24,\"y\":15,\"w\":24,\"h\":15},\"panelIndex\":\"4\",\"embeddableConfig\":{\"title\":\"Disk Usage\"},\"panelRefName\":\"panel_disk_usage\"}]",
11
      "timeRestore": true,
12
      "timeTo": "now",
13
      "timeFrom": "now-24h",
14
      "refreshInterval": {
15
        "pause": false,
16
        "value": 30000
17
      }
18
    }
19
  }]
20
}

1
Collection Intervals:
2
  Critical Metrics:
3
    - CPU, Memory: 30 seconds
4
    - Disk: 60 seconds
5
    - Network: 30 seconds
6

7
  Non-Critical Metrics:
8
    - Process stats: 300 seconds
9
    - System info: 3600 seconds
10

11
Alert Thresholds:
12
  CPU:
13
    - Warning: 70%
14
    - Critical: 85%
15
    - Sustained: 80% for 5 minutes
16

17
  Memory:
18
    - Warning: 75%
19
    - Critical: 90%
20
    - OOM Risk: 95%
21

22
  Disk:
23
    - Warning: 70%
24
    - Critical: 85%
25
    - Emergency: 95%

1
#!/bin/bash
2
# Optimize monitoring performance
3

4
# Use efficient commands
5
# Bad: ps aux | grep process | awk '{print $3}'
6
# Good: pidstat -p $(pgrep process) 1 1 | tail -1 | awk '{print $7}'
7

8
# Cache static information
9
TOTAL_MEM=$(free -b | awk 'NR==2{print $2}')
10
CPU_COUNT=$(nproc)
11

12
# Batch multiple metrics
13
echo "$(date +%s):$(cat /proc/loadavg | cut -d' ' -f1-3):$(free -m | awk 'NR==2{print $3}'):$(df -BG / | awk 'NR==2{print $5}')"

1
<!-- Correlate high CPU with security events -->
2
<rule id="100065" level="13">
3
  <if_sid>100056</if_sid>
4
  <if_matched_sid>5503</if_matched_sid>
5
  <same_source_ip />
6
  <description>High CPU usage coinciding with authentication failures</description>
7
  <mitre>
8
    <id>T1110</id>
9
  </mitre>
10
</rule>
11

12
<!-- Detect crypto mining -->
13
<rule id="100066" level="14">
14
  <if_sid>100056</if_sid>
15
  <time>1:00 am - 6:00 am</time>
16
  <description>Suspicious high CPU usage during off-hours - possible crypto mining</description>
17
  <mitre>
18
    <id>T1496</id>
19
  </mitre>
20
</rule>

1
#!/usr/bin/env python3
2
import json
3
import subprocess
4
import sys
5

6
def handle_alert(alert_file):
7
    with open(alert_file, 'r') as f:
8
        alert = json.load(f)
9

10
    rule_id = alert['rule']['id']
11

12
    if rule_id == '100055':  # High memory
13
        clear_cache()
14
    elif rule_id == '100057':  # Low disk space
15
        clean_logs()
16
    elif rule_id == '100061':  # Sustained high CPU
17
        identify_culprit()
18

19
def clear_cache():
20
    subprocess.run(['sync'])
21
    subprocess.run(['echo', '3'], stdout=open('/proc/sys/vm/drop_caches', 'w'))
22
    print("Memory cache cleared")
23

24
def clean_logs():
25
    subprocess.run(['find', '/var/log', '-name', '*.log', '-mtime', '+30', '-delete'])
26
    subprocess.run(['journalctl', '--vacuum-time=7d'])
27
    print("Old logs cleaned")
28

29
def identify_culprit():
30
    result = subprocess.run(['ps', 'aux', '--sort=-pcpu'], capture_output=True, text=True)
31
    print("Top CPU consumers:")
32
    print(result.stdout.split('\n')[:10])
33

34
if __name__ == "__main__":
35
    handle_alert(sys.argv[1])

1
#!/bin/bash
2
# Generate daily performance report
3

4
REPORT_FILE="/tmp/performance_report_$(date +%Y%m%d).html"
5

6
cat > "$REPORT_FILE" << EOF
7
<!DOCTYPE html>
8
<html>
9
<head>
10
    <title>Daily Performance Report - $(date)</title>
11
    <style>
12
        body { font-family: Arial, sans-serif; }
13
        table { border-collapse: collapse; width: 100%; }
14
        th, td { border: 1px solid #ddd; padding: 8px; text-align: left; }
15
        th { background-color: #4CAF50; color: white; }
16
        .warning { background-color: #fff3cd; }
17
        .critical { background-color: #f8d7da; }
18
    </style>
19
</head>
20
<body>
21
    <h1>Linux Performance Report</h1>
22
    <h2>Date: $(date)</h2>
23

24
    <h3>System Summary</h3>
25
    <table>
26
        <tr>
27
            <th>Metric</th>
28
            <th>Current Value</th>
29
            <th>24h Average</th>
30
            <th>Status</th>
31
        </tr>
32
EOF
33

34
# Add metrics to report
35
# Query Wazuh API for statistics
36

37
echo "</table></body></html>" >> "$REPORT_FILE"
38

39
# Email report
40
mail -s "Daily Performance Report" -a "$REPORT_FILE" admin@company.com < /dev/null