1
apiVersion: v1
2
kind: ConfigMap
3
metadata:
4
  name: spire-server-telemetry-config
5
  namespace: spire-system
6
data:
7
  server.conf: |
8
    server {
9
      bind_address = "0.0.0.0"
10
      bind_port = "8081"
11
      trust_domain = "prod.example.com"
12
      data_dir = "/run/spire/data"
13
      log_level = "INFO"
14

15
      # Enable detailed logging for monitoring
16
      log_format = "json"
17

18
      # Health check endpoints
19
      health_checks {
20
        listener_enabled = true
21
        bind_address = "0.0.0.0"
22
        bind_port = "8080"
23
        live_path = "/live"
24
        ready_path = "/ready"
25
      }
26
    }
27

28
    plugins {
29
      DataStore "sql" {
30
        plugin_data {
31
          database_type = "postgres"
32
          connection_string = "host=postgres port=5432 dbname=spire user=spire password=secret sslmode=require"
33

34
          # Enable connection pooling metrics
35
          connection_pool {
36
            max_open_conns = 100
37
            max_idle_conns = 50
38
            conn_max_lifetime = "1h"
39
          }
40
        }
41
      }
42

43
      NodeAttestor "k8s_psat" {
44
        plugin_data {
45
          cluster = "production"
46
        }
47
      }
48

49
      KeyManager "disk" {
50
        plugin_data {
51
          keys_path = "/run/spire/data/keys"
52
        }
53
      }
54

55
      UpstreamAuthority "disk" {
56
        plugin_data {
57
          cert_file_path = "/run/spire/ca/intermediate.crt"
58
          key_file_path = "/run/spire/ca/intermediate.key"
59
          bundle_file_path = "/run/spire/ca/root.crt"
60
        }
61
      }
62
    }
63

64
    # Comprehensive telemetry configuration
65
    telemetry {
66
      # Prometheus metrics
67
      Prometheus {
68
        host = "0.0.0.0"
69
        port = 9988
70

71
        # Include detailed labels
72
        include_labels = true
73

74
        # Custom metric prefixes
75
        prefix = "spire_server"
76
      }
77

78
      # StatsD for additional metrics aggregation
79
      Statsd {
80
        address = "statsd-exporter.monitoring:9125"
81
        prefix = "spire.server"
82
      }
83

84
      # Enable all available metrics
85
      AllowedPrefixes = []  # Allow all metrics
86
      BlockedPrefixes = []  # Block none
87

88
      # Include detailed labels for better filtering
89
      AllowedLabels = [
90
        "method",
91
        "status_code",
92
        "error_type",
93
        "attestor_type",
94
        "selector_type",
95
        "trust_domain"
96
      ]
97
    }
98
---
99
# Update server deployment with telemetry
100
apiVersion: apps/v1
101
kind: StatefulSet
102
metadata:
103
  name: spire-server
104
  namespace: spire-system
105
spec:
106
  template:
107
    metadata:
108
      annotations:
109
        prometheus.io/scrape: "true"
110
        prometheus.io/port: "9988"
111
        prometheus.io/path: "/metrics"
112
    spec:
113
      containers:
114
        - name: spire-server
115
          ports:
116
            - containerPort: 9988
117
              name: telemetry
118
              protocol: TCP
119
            - containerPort: 8080
120
              name: health
121
              protocol: TCP
122
          livenessProbe:
123
            httpGet:
124
              path: /live
125
              port: 8080
126
            initialDelaySeconds: 30
127
            periodSeconds: 30
128
            timeoutSeconds: 5
129
          readinessProbe:
130
            httpGet:
131
              path: /ready
132
              port: 8080
133
            initialDelaySeconds: 15
134
            periodSeconds: 10
135
            timeoutSeconds: 5

1
apiVersion: v1
2
kind: ConfigMap
3
metadata:
4
  name: spire-agent-telemetry-config
5
  namespace: spire-system
6
data:
7
  agent.conf: |
8
    agent {
9
      data_dir = "/run/spire"
10
      log_level = "INFO"
11
      log_format = "json"
12
      server_address = "spire-server"
13
      server_port = "8081"
14
      socket_path = "/run/spire/sockets/agent.sock"
15
      trust_bundle_path = "/run/spire/bundle/bundle.crt"
16
      trust_domain = "prod.example.com"
17

18
      # Health check configuration
19
      health_checks {
20
        listener_enabled = true
21
        bind_address = "0.0.0.0"
22
        bind_port = "8080"
23
        live_path = "/live"
24
        ready_path = "/ready"
25
      }
26

27
      # Performance settings for monitoring
28
      sync_interval = "30s"
29

30
      # Enable SDS for better observability
31
      sds {
32
        default_svid_name = "default"
33
        default_bundle_name = "ROOTCA"
34
      }
35
    }
36

37
    plugins {
38
      NodeAttestor "k8s_psat" {
39
        plugin_data {
40
          cluster = "production"
41
          token_path = "/run/secrets/tokens/spire-agent"
42
        }
43
      }
44

45
      KeyManager "memory" {
46
        plugin_data {}
47
      }
48

49
      WorkloadAttestor "k8s" {
50
        plugin_data {
51
          # Increase sync interval for monitoring
52
          pod_info_sync_interval = "30s"
53
          skip_kubelet_verification = true
54

55
          # Enable detailed workload labeling
56
          use_new_container_locator = true
57
        }
58
      }
59
    }
60

61
    # Agent telemetry configuration
62
    telemetry {
63
      Prometheus {
64
        host = "0.0.0.0"
65
        port = 9988
66
        prefix = "spire_agent"
67
        include_labels = true
68
      }
69

70
      Statsd {
71
        address = "statsd-exporter.monitoring:9125"
72
        prefix = "spire.agent"
73
      }
74

75
      # Include node and pod information in metrics
76
      AllowedLabels = [
77
        "node_name",
78
        "pod_name",
79
        "pod_namespace",
80
        "workload_selector",
81
        "attestor_type"
82
      ]
83
    }
84
---
85
# Update agent daemonset with telemetry
86
apiVersion: apps/v1
87
kind: DaemonSet
88
metadata:
89
  name: spire-agent
90
  namespace: spire-system
91
spec:
92
  template:
93
    metadata:
94
      annotations:
95
        prometheus.io/scrape: "true"
96
        prometheus.io/port: "9988"
97
        prometheus.io/path: "/metrics"
98
    spec:
99
      containers:
100
        - name: spire-agent
101
          ports:
102
            - containerPort: 9988
103
              name: telemetry
104
              protocol: TCP
105
              hostPort: 9988 # Allow direct access from Prometheus
106
            - containerPort: 8080
107
              name: health
108
              protocol: TCP
109
          livenessProbe:
110
            httpGet:
111
              path: /live
112
              port: 8080
113
            initialDelaySeconds: 30
114
            periodSeconds: 30
115
          readinessProbe:
116
            httpGet:
117
              path: /ready
118
              port: 8080
119
            initialDelaySeconds: 15
120
            periodSeconds: 10

1
apiVersion: v1
2
kind: ConfigMap
3
metadata:
4
  name: prometheus-config
5
  namespace: monitoring
6
data:
7
  prometheus.yml: |
8
    global:
9
      scrape_interval: 15s
10
      evaluation_interval: 15s
11
      external_labels:
12
        cluster: 'production'
13
        region: 'us-east-1'
14

15
    rule_files:
16
      - "/etc/prometheus/rules/*.yml"
17

18
    alerting:
19
      alertmanagers:
20
        - static_configs:
21
            - targets:
22
              - alertmanager:9093
23

24
    scrape_configs:
25
    # SPIRE Server metrics
26
    - job_name: 'spire-server'
27
      kubernetes_sd_configs:
28
      - role: pod
29
        namespaces:
30
          names: ['spire-system']
31

32
      relabel_configs:
33
      # Only scrape pods with the correct labels
34
      - source_labels: [__meta_kubernetes_pod_label_app]
35
        action: keep
36
        regex: spire-server
37

38
      # Add useful labels
39
      - source_labels: [__meta_kubernetes_pod_name]
40
        target_label: pod_name
41
      - source_labels: [__meta_kubernetes_pod_node_name]
42
        target_label: node_name
43
      - source_labels: [__meta_kubernetes_namespace]
44
        target_label: kubernetes_namespace
45

46
      # Set scrape parameters
47
      - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_port]
48
        action: replace
49
        target_label: __address__
50
        regex: (.+)
51
        replacement: ${1}:9988
52

53
      metric_relabel_configs:
54
      # Add server instance information
55
      - source_labels: [__name__]
56
        regex: 'spire_server_.*'
57
        target_label: component
58
        replacement: 'spire-server'
59

60
      # Keep only SPIRE-related metrics
61
      - source_labels: [__name__]
62
        regex: 'spire_server_.*|up|process_.*'
63
        action: keep
64

65
    # SPIRE Agent metrics
66
    - job_name: 'spire-agent'
67
      kubernetes_sd_configs:
68
      - role: pod
69
        namespaces:
70
          names: ['spire-system']
71

72
      relabel_configs:
73
      - source_labels: [__meta_kubernetes_pod_label_app]
74
        action: keep
75
        regex: spire-agent
76

77
      - source_labels: [__meta_kubernetes_pod_name]
78
        target_label: pod_name
79
      - source_labels: [__meta_kubernetes_pod_node_name]
80
        target_label: node_name
81
      - source_labels: [__meta_kubernetes_pod_host_ip]
82
        target_label: __address__
83
        replacement: ${1}:9988
84

85
      metric_relabel_configs:
86
      - source_labels: [__name__]
87
        regex: 'spire_agent_.*'
88
        target_label: component
89
        replacement: 'spire-agent'
90

91
      - source_labels: [__name__]
92
        regex: 'spire_agent_.*|up|process_.*'
93
        action: keep
94

95
    # Workload metrics (applications using SPIFFE)
96
    - job_name: 'spiffe-workloads'
97
      kubernetes_sd_configs:
98
      - role: pod
99

100
      relabel_configs:
101
      # Only scrape pods with SPIFFE annotation
102
      - source_labels: [__meta_kubernetes_pod_annotation_spiffe_io_enabled]
103
        action: keep
104
        regex: true
105

106
      - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
107
        action: keep
108
        regex: true
109

110
      - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path]
111
        action: replace
112
        target_label: __metrics_path__
113
        regex: (.+)
114

115
      - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port]
116
        action: replace
117
        regex: ([^:]+)(?::\d+)?;(\d+)
118
        replacement: $1:$2
119
        target_label: __address__
120

121
      - source_labels: [__meta_kubernetes_namespace]
122
        action: replace
123
        target_label: kubernetes_namespace
124
      - source_labels: [__meta_kubernetes_pod_name]
125
        action: replace
126
        target_label: kubernetes_pod_name
127

128
      metric_relabel_configs:
129
      # Add workload identity information
130
      - source_labels: [__name__]
131
        regex: 'spiffe_.*'
132
        target_label: component
133
        replacement: 'spiffe-workload'
134

135
    # Node Exporter for infrastructure metrics
136
    - job_name: 'node-exporter'
137
      kubernetes_sd_configs:
138
      - role: pod
139
        namespaces:
140
          names: ['monitoring']
141

142
      relabel_configs:
143
      - source_labels: [__meta_kubernetes_pod_label_app]
144
        action: keep
145
        regex: node-exporter
146

147
      - source_labels: [__meta_kubernetes_pod_node_name]
148
        action: replace
149
        target_label: node

1
apiVersion: v1
2
kind: ConfigMap
3
metadata:
4
  name: spiffe-metrics-collector
5
  namespace: spire-system
6
data:
7
  collector.py: |
8
    #!/usr/bin/env python3
9
    import time
10
    import requests
11
    import json
12
    from prometheus_client import start_http_server, Gauge, Counter, Histogram
13
    from kubernetes import client, config
14
    import logging
15

16
    # Set up logging
17
    logging.basicConfig(level=logging.INFO)
18
    logger = logging.getLogger(__name__)
19

20
    # Custom metrics
21
    spiffe_identity_count = Gauge('spiffe_total_identities', 'Total number of SPIFFE identities')
22
    spiffe_expired_certs = Gauge('spiffe_expired_certificates', 'Number of expired certificates')
23
    spiffe_expiring_soon = Gauge('spiffe_certificates_expiring_soon', 'Certificates expiring within 24h')
24
    spiffe_attestation_failures = Counter('spiffe_attestation_failures_total', 'Total attestation failures')
25
    spiffe_svid_fetch_time = Histogram('spiffe_svid_fetch_duration_seconds', 'Time to fetch SVID')
26

27
    class SPIFFEMetricsCollector:
28
        def __init__(self):
29
            self.spire_server_url = "http://spire-server.spire-system:8081"
30

31
        def collect_registration_metrics(self):
32
            """Collect registration entry metrics"""
33
            try:
34
                # Use SPIRE Server API to get entries
35
                response = requests.get(f"{self.spire_server_url}/entries")
36
                if response.status_code == 200:
37
                    entries = response.json().get('entries', [])
38
                    spiffe_identity_count.set(len(entries))
39

40
                    # Count expired and expiring certificates
41
                    now = time.time()
42
                    expired = 0
43
                    expiring_soon = 0
44

45
                    for entry in entries:
46
                        expiry = entry.get('expiry', 0)
47
                        if expiry < now:
48
                            expired += 1
49
                        elif expiry < (now + 86400):  # 24 hours
50
                            expiring_soon += 1
51

52
                    spiffe_expired_certs.set(expired)
53
                    spiffe_expiring_soon.set(expiring_soon)
54

55
            except Exception as e:
56
                logger.error(f"Failed to collect registration metrics: {e}")
57

58
        def collect_workload_metrics(self):
59
            """Collect workload-specific metrics"""
60
            try:
61
                # Load Kubernetes config
62
                config.load_incluster_config()
63
                v1 = client.CoreV1Api()
64

65
                # Get all pods with SPIFFE annotations
66
                pods = v1.list_pod_for_all_namespaces(
67
                    label_selector="spiffe=enabled"
68
                )
69

70
                for pod in pods.items:
71
                    if pod.status.phase == "Running":
72
                        # Simulate SVID fetch time measurement
73
                        # In reality, this would be instrumented in the workload
74
                        fetch_time = self.measure_svid_fetch_time(pod)
75
                        if fetch_time:
76
                            spiffe_svid_fetch_time.observe(fetch_time)
77

78
            except Exception as e:
79
                logger.error(f"Failed to collect workload metrics: {e}")
80

81
        def measure_svid_fetch_time(self, pod):
82
            """Measure time to fetch SVID for a pod"""
83
            # This is a placeholder - in production, instrument your workloads
84
            return 0.1  # Mock 100ms fetch time
85

86
        def run(self):
87
            logger.info("Starting SPIFFE metrics collector")
88
            while True:
89
                try:
90
                    self.collect_registration_metrics()
91
                    self.collect_workload_metrics()
92
                    time.sleep(30)  # Collect every 30 seconds
93
                except Exception as e:
94
                    logger.error(f"Error in collection cycle: {e}")
95
                    time.sleep(10)
96

97
    if __name__ == '__main__':
98
        # Start Prometheus metrics server
99
        start_http_server(8000)
100
        logger.info("Metrics server started on port 8000")
101

102
        # Start collector
103
        collector = SPIFFEMetricsCollector()
104
        collector.run()
105
---
106
apiVersion: apps/v1
107
kind: Deployment
108
metadata:
109
  name: spiffe-metrics-collector
110
  namespace: spire-system
111
spec:
112
  replicas: 1
113
  selector:
114
    matchLabels:
115
      app: spiffe-metrics-collector
116
  template:
117
    metadata:
118
      labels:
119
        app: spiffe-metrics-collector
120
      annotations:
121
        prometheus.io/scrape: "true"
122
        prometheus.io/port: "8000"
123
    spec:
124
      serviceAccountName: spiffe-metrics-collector
125
      containers:
126
        - name: collector
127
          image: python:3.9-slim
128
          command: ["python", "/app/collector.py"]
129
          ports:
130
            - containerPort: 8000
131
              name: metrics
132
          env:
133
            - name: PYTHONUNBUFFERED
134
              value: "1"
135
          volumeMounts:
136
            - name: app
137
              mountPath: /app
138
          resources:
139
            requests:
140
              memory: "128Mi"
141
              cpu: "100m"
142
            limits:
143
              memory: "256Mi"
144
              cpu: "200m"
145
      volumes:
146
        - name: app
147
          configMap:
148
            name: spiffe-metrics-collector
149
            defaultMode: 0755

1
apiVersion: monitoring.coreos.com/v1
2
kind: PrometheusRule
3
metadata:
4
  name: spire-critical-alerts
5
  namespace: spire-system
6
spec:
7
  groups:
8
    - name: spire.critical
9
      interval: 30s
10
      rules:
11
        # Server Availability
12
        - alert: SPIREServerDown
13
          expr: up{job="spire-server"} == 0
14
          for: 2m
15
          labels:
16
            severity: critical
17
            component: spire-server
18
          annotations:
19
            summary: "SPIRE Server instance is down"
20
            description: "SPIRE Server {{ $labels.instance }} has been down for more than 2 minutes. This affects workload identity issuance."
21
            runbook_url: "https://wiki.company.com/spire-runbooks#server-down"
22

23
        # Database Connectivity
24
        - alert: SPIREDatabaseConnectionFailure
25
          expr: spire_server_datastore_connections_active == 0
26
          for: 5m
27
          labels:
28
            severity: critical
29
            component: datastore
30
          annotations:
31
            summary: "SPIRE Server cannot connect to database"
32
            description: "SPIRE Server {{ $labels.instance }} has no active database connections for 5 minutes."
33

34
        # High Error Rate
35
        - alert: SPIREHighErrorRate
36
          expr: |
37
            sum(rate(spire_server_api_errors_total[5m])) by (instance)
38
            / sum(rate(spire_server_api_requests_total[5m])) by (instance) > 0.05
39
          for: 10m
40
          labels:
41
            severity: warning
42
            component: api
43
          annotations:
44
            summary: "High error rate in SPIRE Server API"
45
            description: "SPIRE Server {{ $labels.instance }} API error rate is {{ $value | humanizePercentage }} over the last 10 minutes."
46

47
        # Certificate Expiry
48
        - alert: SPIRECACertificateExpiringSoon
49
          expr: |
50
            (spire_server_ca_certificate_expiry_timestamp - time()) / 86400 < 30
51
          for: 1h
52
          labels:
53
            severity: warning
54
            component: certificates
55
          annotations:
56
            summary: "SPIRE CA certificate expiring soon"
57
            description: "SPIRE CA certificate will expire in {{ $value }} days. Plan for rotation."
58

59
        - alert: SPIRECACertificateExpired
60
          expr: |
61
            (spire_server_ca_certificate_expiry_timestamp - time()) < 0
62
          for: 1m
63
          labels:
64
            severity: critical
65
            component: certificates
66
          annotations:
67
            summary: "SPIRE CA certificate has expired"
68
            description: "SPIRE CA certificate has expired. Immediate action required."
69

70
        # Agent Issues
71
        - alert: SPIREAgentSyncFailures
72
          expr: |
73
            rate(spire_agent_sync_failures_total[5m]) > 0.1
74
          for: 15m
75
          labels:
76
            severity: warning
77
            component: agent
78
          annotations:
79
            summary: "High agent sync failure rate"
80
            description: "SPIRE Agent {{ $labels.instance }} sync failure rate is {{ $value }} failures/second."
81

82
        - alert: SPIREAgentDisconnected
83
          expr: |
84
            (time() - spire_agent_last_sync_timestamp) > 300
85
          for: 5m
86
          labels:
87
            severity: critical
88
            component: agent
89
          annotations:
90
            summary: "SPIRE Agent disconnected"
91
            description: "SPIRE Agent {{ $labels.instance }} hasn't synced for {{ $value }} seconds."
92

93
        # Security Alerts
94
        - alert: SPIREUnauthorizedAccessSpike
95
          expr: |
96
            sum(rate(spire_server_api_unauthorized_total[5m])) > 1
97
          for: 5m
98
          labels:
99
            severity: warning
100
            component: security
101
          annotations:
102
            summary: "Spike in unauthorized access attempts"
103
            description: "Unauthorized access attempts: {{ $value }} requests/second to SPIRE Server."
104

105
        - alert: SPIREAttestationFailureSpike
106
          expr: |
107
            sum(rate(spire_server_attestation_failures_total[5m])) by (attestor_type) > 0.5
108
          for: 10m
109
          labels:
110
            severity: warning
111
            component: security
112
          annotations:
113
            summary: "High attestation failure rate"
114
            description: "Attestation failures for {{ $labels.attestor_type }}: {{ $value }} failures/second."
115

116
        # Performance Alerts
117
        - alert: SPIREHighLatency
118
          expr: |
119
            histogram_quantile(0.95,
120
              sum(rate(spire_server_api_request_duration_seconds_bucket[5m])) by (le, method)
121
            ) > 5
122
          for: 15m
123
          labels:
124
            severity: warning
125
            component: performance
126
          annotations:
127
            summary: "High API latency"
128
            description: "95th percentile latency for {{ $labels.method }} is {{ $value }}s."
129

130
        - alert: SPIREHighMemoryUsage
131
          expr: |
132
            process_resident_memory_bytes{job="spire-server"} / (1024*1024*1024) > 4
133
          for: 15m
134
          labels:
135
            severity: warning
136
            component: resources
137
          annotations:
138
            summary: "High memory usage"
139
            description: "SPIRE Server {{ $labels.instance }} using {{ $value }}GB of memory."
140

141
        # Capacity Planning
142
        - alert: SPIREEntryCountHigh
143
          expr: |
144
            spire_server_registration_entries > 50000
145
          for: 30m
146
          labels:
147
            severity: warning
148
            component: capacity
149
          annotations:
150
            summary: "High number of registration entries"
151
            description: "SPIRE Server has {{ $value }} registration entries. Consider capacity planning."