1
#!/bin/bash
2
# DNS monitoring script
3
DOMAINS=("app.example.com" "api.example.com")
4
RECORD_TYPES=("A" "AAAA" "CNAME" "TXT")
5

6
for domain in "${DOMAINS[@]}"; do
7
    for type in "${RECORD_TYPES[@]}"; do
8
        result=$(dig +short $domain $type)
9
        if [ -z "$result" ]; then
10
            echo "WARNING: No $type record found for $domain"
11
        else
12
            echo "OK: $domain $type = $result"
13
        fi
14
    done
15
done

1
# Prometheus configuration for DNS monitoring
2
- job_name: "dns_monitoring"
3
  metrics_path: /probe
4
  params:
5
    module: [dns]
6
  static_configs:
7
    - targets:
8
        - app.example.com
9
        - api.example.com
10
  relabel_configs:
11
    - source_labels: [__address__]
12
      target_label: __param_target
13
    - source_labels: [__param_target]
14
      target_label: instance
15
    - target_label: __address__
16
      replacement: blackbox-exporter:9115

1
#!/bin/bash
2
# Certificate monitoring script
3
check_certificate() {
4
    local host=$1
5
    local port=${2:-443}
6

7
    # Check certificate expiry
8
    expiry=$(echo | openssl s_client -servername $host -connect $host:$port 2>/dev/null | \
9
             openssl x509 -noout -enddate 2>/dev/null | cut -d= -f2)
10

11
    # Check TLS version
12
    tls_version=$(echo | openssl s_client -servername $host -connect $host:$port 2>/dev/null | \
13
                  grep "Protocol" | awk '{print $3}')
14

15
    echo "Host: $host"
16
    echo "Expiry: $expiry"
17
    echo "TLS Version: $tls_version"
18

19
    # Check if certificate expires within 30 days
20
    expiry_epoch=$(date -d "$expiry" +%s)
21
    current_epoch=$(date +%s)
22
    days_left=$(( ($expiry_epoch - $current_epoch) / 86400 ))
23

24
    if [ $days_left -lt 30 ]; then
25
        echo "WARNING: Certificate expires in $days_left days"
26
    fi
27
}
28

29
# Monitor multiple endpoints
30
check_certificate "app.example.com" 443
31
check_certificate "api.example.com" 8443

1
# Blackbox exporter configuration
2
modules:
3
  https_2xx:
4
    prober: http
5
    timeout: 5s
6
    http:
7
      valid_status_codes: []
8
      valid_http_versions: ["HTTP/1.1", "HTTP/2.0"]
9
      tls_config:
10
        insecure_skip_verify: false
11
        min_version: "TLS12"
12
      preferred_ip_protocol: "ip4"

1
#!/bin/bash
2
# Service monitoring script
3

4
# HTTP/HTTPS monitoring
5
check_http() {
6
    local url=$1
7
    local expected_code=${2:-200}
8

9
    response=$(curl -s -o /dev/null -w "%{http_code}" $url)
10
    if [ "$response" == "$expected_code" ]; then
11
        echo "OK: $url returned $response"
12
    else
13
        echo "ERROR: $url returned $response (expected $expected_code)"
14
    fi
15
}
16

17
# TCP port monitoring
18
check_tcp() {
19
    local host=$1
20
    local port=$2
21

22
    nc -z -v -w5 $host $port &>/dev/null
23
    if [ $? -eq 0 ]; then
24
        echo "OK: $host:$port is reachable"
25
    else
26
        echo "ERROR: $host:$port is not reachable"
27
    fi
28
}
29

30
# SSH monitoring
31
check_ssh() {
32
    local host=$1
33
    local port=${2:-22}
34

35
    timeout 5 ssh -o BatchMode=yes -o ConnectTimeout=5 $host exit 2>/dev/null
36
    if [ $? -eq 0 ]; then
37
        echo "OK: SSH to $host:$port successful"
38
    else
39
        echo "ERROR: SSH to $host:$port failed"
40
    fi
41
}
42

43
# Internal checks
44
check_http "http://localhost:8080/health"
45
check_tcp "localhost" 5432  # PostgreSQL
46
check_ssh "localhost"
47

48
# External checks
49
check_http "https://app.example.com"
50
check_tcp "app.example.com" 443

1
FROM alpine:latest
2

3
# Install dependencies
4
RUN apk add --no-cache curl
5

6
# Add healthcheck
7
HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \
8
  CMD curl -f http://localhost:8080/health || exit 1
9

10
# Application setup
11
COPY app /app
12
EXPOSE 8080
13
CMD ["/app/server"]

1
#!/bin/bash
2
# Container health monitoring script
3

4
# Check container health status
5
check_container_health() {
6
    local container=$1
7

8
    health_status=$(podman inspect $container --format='{{.State.Health.Status}}')
9
    case $health_status in
10
        "healthy")
11
            echo "OK: Container $container is healthy"
12
            ;;
13
        "unhealthy")
14
            echo "ERROR: Container $container is unhealthy"
15
            # Get last health check logs
16
            podman inspect $container --format='{{json .State.Health.Log}}' | jq '.'
17
            ;;
18
        "starting")
19
            echo "INFO: Container $container health check is starting"
20
            ;;
21
        *)
22
            echo "WARNING: Container $container has no health check"
23
            ;;
24
    esac
25
}
26

27
# Monitor lifecycle events
28
monitor_events() {
29
    podman events --filter event=health_status --format json | while read line; do
30
        container=$(echo $line | jq -r '.Actor.Attributes.name')
31
        status=$(echo $line | jq -r '.Actor.Attributes.health_status')
32
        timestamp=$(echo $line | jq -r '.time')
33

34
        echo "[$timestamp] Container: $container, Health: $status"
35

36
        # Alert on unhealthy containers
37
        if [ "$status" == "unhealthy" ]; then
38
            send_alert "Container $container is unhealthy"
39
        fi
40
    done
41
}
42

43
# Check all running containers
44
for container in $(podman ps -q); do
45
    check_container_health $container
46
done

1
#!/bin/bash
2
# Resource monitoring script
3

4
# Real-time container stats
5
monitor_container_resources() {
6
    local container=$1
7
    local threshold_cpu=80
8
    local threshold_memory=90
9

10
    # Get container stats
11
    stats=$(podman stats --no-stream --format json $container)
12

13
    # Parse metrics
14
    cpu_percent=$(echo $stats | jq -r '.[0].CPU' | sed 's/%//')
15
    memory_percent=$(echo $stats | jq -r '.[0].MemPerc' | sed 's/%//')
16
    memory_usage=$(echo $stats | jq -r '.[0].MemUsage')
17
    net_io=$(echo $stats | jq -r '.[0].NetIO')
18
    block_io=$(echo $stats | jq -r '.[0].BlockIO')
19

20
    echo "Container: $container"
21
    echo "  CPU: ${cpu_percent}%"
22
    echo "  Memory: ${memory_percent}% ($memory_usage)"
23
    echo "  Network I/O: $net_io"
24
    echo "  Block I/O: $block_io"
25

26
    # Alert on high usage
27
    if (( $(echo "$cpu_percent > $threshold_cpu" | bc -l) )); then
28
        echo "WARNING: CPU usage above threshold"
29
    fi
30

31
    if (( $(echo "$memory_percent > $threshold_memory" | bc -l) )); then
32
        echo "WARNING: Memory usage above threshold"
33
    fi
34
}
35

36
# Check storage usage
37
check_storage() {
38
    # Container storage
39
    podman system df
40

41
    # Image storage
42
    echo "Image Storage:"
43
    podman image ls --format "table {{.Repository}}:{{.Tag}}\t{{.Size}}"
44

45
    # Volume storage
46
    echo "Volume Storage:"
47
    podman volume ls --format "table {{.Name}}\t{{.Driver}}\t{{.Scope}}"
48
}
49

50
# Monitor all containers
51
for container in $(podman ps --format "{{.Names}}"); do
52
    monitor_container_resources $container
53
done
54

55
check_storage

1
# cAdvisor alternative for Podman
2
- job_name: "podman"
3
  static_configs:
4
    - targets: ["localhost:9090"]
5
  metric_relabel_configs:
6
    - source_labels: [__name__]
7
      regex: "container_.*"
8
      action: keep

1
# Fluentd configuration for container logs
2
<source>
3
@type forward
4
port 24224
5
bind 0.0.0.0
6
</source>
7

8
<filter docker.**>
9
@type parser
10
key_name log
11
format json
12
reserve_data true
13
</filter>
14

15
<match docker.**>
16
@type elasticsearch
17
host elasticsearch
18
port 9200
19
logstash_format true
20
logstash_prefix container
21
<buffer>
22
@type file
23
path /var/log/fluentd-buffers/containers.buffer
24
flush_mode interval
25
flush_interval 10s
26
</buffer>
27
</match>

1
#!/bin/bash
2
# Network performance monitoring
3

4
# Monitor container network performance
5
check_container_network() {
6
    local container=$1
7
    local target=$2
8

9
    # Get container PID
10
    pid=$(podman inspect -f '{{.State.Pid}}' $container)
11

12
    # Enter container network namespace
13
    nsenter -t $pid -n ping -c 10 -i 0.2 $target > /tmp/ping_results.txt
14

15
    # Parse results
16
    packet_loss=$(grep "packet loss" /tmp/ping_results.txt | awk -F',' '{print $3}' | awk '{print $1}')
17
    avg_latency=$(grep "rtt min/avg/max" /tmp/ping_results.txt | awk -F'/' '{print $5}')
18

19
    echo "Container: $container -> $target"
20
    echo "  Packet Loss: $packet_loss"
21
    echo "  Average Latency: ${avg_latency}ms"
22

23
    # Throughput test using iperf3
24
    if command -v iperf3 &> /dev/null; then
25
        nsenter -t $pid -n iperf3 -c $target -t 10 -J > /tmp/iperf_results.json
26
        throughput=$(jq -r '.end.sum_sent.bits_per_second' /tmp/iperf_results.json)
27
        throughput_mbps=$(echo "scale=2; $throughput / 1000000" | bc)
28
        echo "  Throughput: ${throughput_mbps} Mbps"
29
    fi
30
}
31

32
# Monitor CNI plugin metrics
33
monitor_cni() {
34
    # Check bridge networks
35
    podman network ls --format "table {{.Name}}\t{{.Driver}}\t{{.Subnets}}"
36

37
    # Inspect network details
38
    for network in $(podman network ls -q); do
39
        echo "Network: $network"
40
        podman network inspect $network | jq '.[] | {name: .name, driver: .driver, subnets: .subnets}'
41
    done
42
}

1
#!/bin/bash
2
# Security and compliance monitoring
3

4
# Vulnerability scanning with Trivy
5
scan_container_image() {
6
    local image=$1
7

8
    echo "Scanning image: $image"
9
    trivy image --severity HIGH,CRITICAL --format json $image > /tmp/scan_results.json
10

11
    # Parse results
12
    high_vulns=$(jq '[.Results[].Vulnerabilities[] | select(.Severity=="HIGH")] | length' /tmp/scan_results.json)
13
    critical_vulns=$(jq '[.Results[].Vulnerabilities[] | select(.Severity=="CRITICAL")] | length' /tmp/scan_results.json)
14

15
    echo "  Critical vulnerabilities: $critical_vulns"
16
    echo "  High vulnerabilities: $high_vulns"
17

18
    if [ $critical_vulns -gt 0 ]; then
19
        echo "ERROR: Critical vulnerabilities found!"
20
        jq '.Results[].Vulnerabilities[] | select(.Severity=="CRITICAL") | {id: .VulnerabilityID, package: .PkgName, severity: .Severity}' /tmp/scan_results.json
21
    fi
22
}
23

24
# Policy compliance checks
25
check_security_policies() {
26
    echo "Checking security policies..."
27

28
    # Check for running containers as root
29
    echo "Containers running as root:"
30
    podman ps --format json | jq -r '.[] | select(.User == "root") | .Names'
31

32
    # Check for containers with privileged access
33
    echo "Privileged containers:"
34
    podman ps --format json | jq -r '.[] | select(.IsPrivileged == true) | .Names'
35

36
    # Check image signatures
37
    echo "Unsigned images:"
38
    for image in $(podman image ls -q); do
39
        if ! podman image trust show $image &>/dev/null; then
40
            echo "  - $image"
41
        fi
42
    done
43

44
    # Certificate expiry monitoring
45
    echo "Certificate expiry check:"
46
    find /etc/containers/certs.d -name "*.crt" -type f | while read cert; do
47
        expiry=$(openssl x509 -enddate -noout -in "$cert" | cut -d= -f2)
48
        expiry_epoch=$(date -d "$expiry" +%s)
49
        current_epoch=$(date +%s)
50
        days_left=$(( ($expiry_epoch - $current_epoch) / 86400 ))
51

52
        if [ $days_left -lt 30 ]; then
53
            echo "  WARNING: $cert expires in $days_left days"
54
        fi
55
    done
56
}
57

58
# Scan all running container images
59
for image in $(podman ps --format "{{.Image}}" | sort -u); do
60
    scan_container_image $image
61
done
62

63
check_security_policies

1
#!/bin/bash
2
# Pod-level monitoring
3

4
# Monitor pod health
5
monitor_pod() {
6
    local pod=$1
7

8
    echo "Pod: $pod"
9

10
    # Get pod status
11
    pod_status=$(podman pod inspect $pod --format json | jq -r '.[0].State')
12
    echo "  Status: $pod_status"
13

14
    # Get containers in pod
15
    containers=$(podman pod inspect $pod --format json | jq -r '.[0].Containers[].Name')
16

17
    # Aggregate resource usage
18
    total_cpu=0
19
    total_memory=0
20
    unhealthy_count=0
21

22
    for container in $containers; do
23
        # Get container stats
24
        stats=$(podman stats --no-stream --format json $container 2>/dev/null)
25
        if [ $? -eq 0 ]; then
26
            cpu=$(echo $stats | jq -r '.[0].CPU' | sed 's/%//')
27
            memory=$(echo $stats | jq -r '.[0].MemPerc' | sed 's/%//')
28

29
            total_cpu=$(echo "$total_cpu + $cpu" | bc)
30
            total_memory=$(echo "$total_memory + $memory" | bc)
31

32
            # Check health
33
            health=$(podman inspect $container --format='{{.State.Health.Status}}' 2>/dev/null)
34
            if [ "$health" == "unhealthy" ]; then
35
                ((unhealthy_count++))
36
            fi
37
        fi
38
    done
39

40
    echo "  Total CPU Usage: ${total_cpu}%"
41
    echo "  Total Memory Usage: ${total_memory}%"
42
    echo "  Unhealthy Containers: $unhealthy_count"
43

44
    # Generate Kubernetes-compatible YAML for documentation
45
    podman generate kube $pod > /tmp/${pod}_kube.yaml
46
    echo "  Kubernetes YAML generated: /tmp/${pod}_kube.yaml"
47
}
48

49
# Create pod with monitoring
50
create_monitored_pod() {
51
    local pod_name=$1
52

53
    # Create pod
54
    podman pod create --name $pod_name \
55
        --label monitoring=enabled \
56
        --label environment=production
57

58
    # Add containers to pod
59
    podman run -d --pod $pod_name \
60
        --name ${pod_name}-app \
61
        --health-cmd="curl -f http://localhost:8080/health || exit 1" \
62
        --health-interval=30s \
63
        myapp:latest
64

65
    podman run -d --pod $pod_name \
66
        --name ${pod_name}-sidecar \
67
        --health-cmd="nc -z localhost 9090 || exit 1" \
68
        --health-interval=30s \
69
        monitoring-sidecar:latest
70
}
71

72
# Monitor all pods
73
for pod in $(podman pod ls -q); do
74
    monitor_pod $pod
75
done

1
#!/bin/bash
2
# Host metrics monitoring
3

4
# Comprehensive host monitoring
5
monitor_host_metrics() {
6
    echo "=== Host System Metrics ==="
7

8
    # CPU metrics
9
    echo "CPU Usage:"
10
    top -bn1 | grep "Cpu(s)" | awk '{print "  User: " $2 "%, System: " $4 "%, Idle: " $8 "%"}'
11

12
    # Memory metrics
13
    echo "Memory Usage:"
14
    free -h | awk '/^Mem:/ {print "  Total: " $2 ", Used: " $3 ", Free: " $4 ", Available: " $7}'
15

16
    # Disk metrics
17
    echo "Disk Usage:"
18
    df -h | grep -E "^/dev/" | awk '{print "  " $1 ": " $5 " used (" $3 "/" $2 ")"}'
19

20
    # Container storage specific
21
    echo "Container Storage:"
22
    podman system df
23

24
    # Network metrics
25
    echo "Network Usage:"
26
    for interface in $(ip -o link show | awk -F': ' '{print $2}' | grep -v lo); do
27
        rx_bytes=$(cat /sys/class/net/$interface/statistics/rx_bytes)
28
        tx_bytes=$(cat /sys/class/net/$interface/statistics/tx_bytes)
29
        rx_mb=$(echo "scale=2; $rx_bytes / 1024 / 1024" | bc)
30
        tx_mb=$(echo "scale=2; $tx_bytes / 1024 / 1024" | bc)
31
        echo "  $interface: RX: ${rx_mb}MB, TX: ${tx_mb}MB"
32
    done
33

34
    # Load average
35
    echo "Load Average:"
36
    uptime | awk -F'load average:' '{print "  " $2}'
37
}
38

39
# Correlate with container metrics
40
correlate_metrics() {
41
    # Get total container resource usage
42
    container_cpu=$(podman stats --no-stream --format "{{.CPU}}" | sed 's/%//g' | awk '{sum+=$1} END {print sum}')
43
    container_memory=$(podman stats --no-stream --format "{{.MemPerc}}" | sed 's/%//g' | awk '{sum+=$1} END {print sum}')
44

45
    # Get host usage
46
    host_cpu=$(top -bn1 | grep "Cpu(s)" | awk '{print 100 - $8}' | sed 's/%,//')
47
    host_memory=$(free | awk '/^Mem:/ {print ($3/$2) * 100}')
48

49
    echo "=== Resource Correlation ==="
50
    echo "Container CPU Usage: ${container_cpu}%"
51
    echo "Host CPU Usage: ${host_cpu}%"
52
    echo "Container Memory Usage: ${container_memory}%"
53
    echo "Host Memory Usage: ${host_memory}%"
54

55
    # Calculate overhead
56
    cpu_overhead=$(echo "scale=2; $host_cpu - $container_cpu" | bc)
57
    memory_overhead=$(echo "scale=2; $host_memory - $container_memory" | bc)
58

59
    echo "System Overhead:"
60
    echo "  CPU: ${cpu_overhead}%"
61
    echo "  Memory: ${memory_overhead}%"
62
}
63

64
# Prometheus node exporter integration
65
setup_node_exporter() {
66
    # Run node exporter in container
67
    podman run -d \
68
        --name node-exporter \
69
        --net host \
70
        --pid host \
71
        --volume /:/host:ro,rslave \
72
        quay.io/prometheus/node-exporter:latest \
73
        --path.rootfs=/host
74
}
75

76
monitor_host_metrics
77
correlate_metrics

1
# docker-compose.yml for monitoring stack
2
version: "3.8"
3

4
services:
5
  prometheus:
6
    image: prom/prometheus:latest
7
    volumes:
8
      - ./prometheus.yml:/etc/prometheus/prometheus.yml
9
      - prometheus-data:/prometheus
10
    ports:
11
      - "9090:9090"
12
    command:
13
      - "--config.file=/etc/prometheus/prometheus.yml"
14
      - "--storage.tsdb.path=/prometheus"
15

16
  grafana:
17
    image: grafana/grafana:latest
18
    volumes:
19
      - grafana-data:/var/lib/grafana
20
      - ./grafana/dashboards:/etc/grafana/provisioning/dashboards
21
      - ./grafana/datasources:/etc/grafana/provisioning/datasources
22
    environment:
23
      - GF_SECURITY_ADMIN_PASSWORD=admin
24
      - GF_USERS_ALLOW_SIGN_UP=false
25
    ports:
26
      - "3000:3000"
27

28
  alertmanager:
29
    image: prom/alertmanager:latest
30
    volumes:
31
      - ./alertmanager.yml:/etc/alertmanager/alertmanager.yml
32
      - alertmanager-data:/alertmanager
33
    ports:
34
      - "9093:9093"
35

36
  blackbox-exporter:
37
    image: prom/blackbox-exporter:latest
38
    volumes:
39
      - ./blackbox.yml:/config/blackbox.yml
40
    ports:
41
      - "9115:9115"
42
    command:
43
      - "--config.file=/config/blackbox.yml"
44

45
  loki:
46
    image: grafana/loki:latest
47
    volumes:
48
      - ./loki-config.yml:/etc/loki/local-config.yaml
49
      - loki-data:/loki
50
    ports:
51
      - "3100:3100"
52
    command: -config.file=/etc/loki/local-config.yaml
53

54
  promtail:
55
    image: grafana/promtail:latest
56
    volumes:
57
      - ./promtail-config.yml:/etc/promtail/config.yml
58
      - /var/log:/var/log:ro
59
      - /var/lib/containers:/var/lib/containers:ro
60
    command: -config.file=/etc/promtail/config.yml
61

62
volumes:
63
  prometheus-data:
64
  grafana-data:
65
  alertmanager-data:
66
  loki-data: