Kubernetes API Authentication & RBAC: Production-Ready Security Guide#

1
apiVersion: v1
2
kind: ServiceAccount
3
metadata:
4
  name: app-service-account
5
  namespace: production
6
  annotations:
7
    # Disable automatic token mounting for enhanced security
8
    automountServiceAccountToken: "false"
9
    # Add custom annotations for tracking
10
    purpose: "application-api-access"
11
    owner: "platform-team"
12
---
13
# secret.yaml - Manual token creation (K8s 1.24+)
14
apiVersion: v1
15
kind: Secret
16
metadata:
17
  name: app-sa-token
18
  namespace: production
19
  annotations:
20
    kubernetes.io/service-account.name: app-service-account
21
type: kubernetes.io/service-account-token

1
apiVersion: v1
2
kind: Namespace
3
metadata:
4
  name: production
5
  labels:
6
    environment: production
7
    compliance: pci-dss
8
---
9
# role-definitions.yaml
10
apiVersion: rbac.authorization.k8s.io/v1
11
kind: Role
12
metadata:
13
  namespace: production
14
  name: app-reader
15
rules:
16
- apiGroups: [""]
17
  resources: ["pods", "services"]
18
  verbs: ["get", "list", "watch"]
19
- apiGroups: ["apps"]
20
  resources: ["deployments", "replicasets"]
21
  verbs: ["get", "list"]
22
- apiGroups: [""]
23
  resources: ["configmaps"]
24
  resourceNames: ["app-config"]
25
  verbs: ["get"]
26
---
27
apiVersion: rbac.authorization.k8s.io/v1
28
kind: Role
29
metadata:
30
  namespace: production
31
  name: app-writer
32
rules:
33
- apiGroups: [""]
34
  resources: ["pods"]
35
  verbs: ["get", "list", "create", "update", "patch", "delete"]
36
- apiGroups: ["apps"]
37
  resources: ["deployments"]
38
  verbs: ["get", "list", "create", "update", "patch"]
39
- apiGroups: ["autoscaling"]
40
  resources: ["horizontalpodautoscalers"]
41
  verbs: ["get", "list", "create", "update", "patch"]
42
---
43
# role-bindings.yaml
44
apiVersion: rbac.authorization.k8s.io/v1
45
kind: RoleBinding
46
metadata:
47
  name: read-pods-services
48
  namespace: production
49
subjects:
50
- kind: ServiceAccount
51
  name: app-service-account
52
  namespace: production
53
- kind: Group
54
  name: "developers"
55
  apiGroup: rbac.authorization.k8s.io
56
roleRef:
57
  kind: Role
58
  name: app-reader
59
  apiGroup: rbac.authorization.k8s.io

1
apiVersion: rbac.authorization.k8s.io/v1
2
kind: ClusterRole
3
metadata:
4
  name: monitoring-reader
5
rules:
6
- apiGroups: [""]
7
  resources: ["pods", "nodes", "namespaces"]
8
  verbs: ["get", "list"]
9
- apiGroups: ["metrics.k8s.io"]
10
  resources: ["pods", "nodes"]
11
  verbs: ["get", "list"]
12
- apiGroups: [""]
13
  resources: ["events"]
14
  verbs: ["get", "list", "watch"]
15
---
16
apiVersion: rbac.authorization.k8s.io/v1
17
kind: ClusterRoleBinding
18
metadata:
19
  name: monitoring-reader-binding
20
roleRef:
21
  apiGroup: rbac.authorization.k8s.io
22
  kind: ClusterRole
23
  name: monitoring-reader
24
subjects:
25
- kind: ServiceAccount
26
  name: prometheus
27
  namespace: monitoring

1
apiVersion: rbac.authorization.k8s.io/v1
2
kind: ClusterRole
3
metadata:
4
  name: custom-view
5
  labels:
6
    # Aggregate to default view role
7
    rbac.authorization.k8s.io/aggregate-to-view: "true"
8
rules:
9
- apiGroups: ["custom.io"]
10
  resources: ["customresources"]
11
  verbs: ["get", "list", "watch"]
12
---
13
# conditional-rbac.yaml
14
apiVersion: rbac.authorization.k8s.io/v1
15
kind: Role
16
metadata:
17
  namespace: production
18
  name: deployment-manager
19
rules:
20
# Allow scaling but not deletion
21
- apiGroups: ["apps"]
22
  resources: ["deployments/scale"]
23
  verbs: ["update", "patch"]
24
# Read-only for statefulsets
25
- apiGroups: ["apps"]
26
  resources: ["statefulsets"]
27
  verbs: ["get", "list", "watch"]
28
# Full access to specific deployment
29
- apiGroups: ["apps"]
30
  resources: ["deployments"]
31
  resourceNames: ["frontend", "backend"]
32
  verbs: ["*"]

1
#!/bin/bash
2
echo "=== Service Account Audit ==="
3
echo
4

5
# List all service accounts with their tokens
6
for ns in $(kubectl get ns -o jsonpath='{.items[*].metadata.name}'); do
7
    echo "Namespace: $ns"
8
    kubectl get sa -n $ns -o json | jq -r '.items[] |
9
        select(.metadata.name != "default") |
10
        "\(.metadata.name): Tokens=\(.secrets | length)"'
11
    echo
12
done
13

14
# Find service accounts with cluster-admin
15
echo "=== Service Accounts with cluster-admin ==="
16
kubectl get clusterrolebindings -o json | jq -r '
17
    .items[] |
18
    select(.roleRef.name == "cluster-admin") |
19
    .subjects[]? |
20
    select(.kind == "ServiceAccount") |
21
    "\(.namespace)/\(.name)"'
22

23
# Check for wildcard permissions
24
echo -e "\n=== Roles with Wildcard Permissions ==="
25
kubectl get roles,clusterroles -A -o json | jq -r '
26
    .items[] |
27
    select(.rules[]? |
28
        (.verbs | contains(["*"])) or
29
        (.resources | contains(["*"])) or
30
        (.apiGroups | contains(["*"]))) |
31
    "\(.kind)/\(.metadata.namespace // "cluster")/\(.metadata.name)"'

1
apiVersion: batch/v1
2
kind: CronJob
3
metadata:
4
  name: token-rotator
5
  namespace: security
6
spec:
7
  schedule: "0 2 * * 0"  # Weekly on Sunday at 2 AM
8
  jobTemplate:
9
    spec:
10
      template:
11
        spec:
12
          serviceAccountName: token-manager
13
          containers:
14
          - name: rotator
15
            image: bitnami/kubectl:latest
16
            command:
17
            - /bin/bash
18
            - -c
19
            - |
20
              # List all service account tokens older than 30 days
21
              for ns in $(kubectl get ns -o name | cut -d/ -f2); do
22
                for secret in $(kubectl get secrets -n $ns -o json | \
23
                  jq -r '.items[] |
24
                  select(.type == "kubernetes.io/service-account-token") |
25
                  select(.metadata.creationTimestamp | fromdateiso8601 < (now - 2592000)) |
26
                  .metadata.name'); do
27

28
                  echo "Rotating token: $ns/$secret"
29
                  # Delete old token (will be auto-recreated)
30
                  kubectl delete secret $secret -n $ns
31
                done
32
              done
33
          restartPolicy: OnFailure

1
apiVersion: v1
2
kind: ConfigMap
3
metadata:
4
  name: kube-apiserver-config
5
  namespace: kube-system
6
data:
7
  config.yaml: |
8
    apiVersion: kubeadm.k8s.io/v1beta3
9
    kind: ClusterConfiguration
10
    apiServer:
11
      extraArgs:
12
        oidc-issuer-url: "https://keycloak.example.com/auth/realms/kubernetes"
13
        oidc-client-id: "kubernetes"
14
        oidc-username-claim: "preferred_username"
15
        oidc-username-prefix: "oidc:"
16
        oidc-groups-claim: "groups"
17
        oidc-groups-prefix: "oidc:"
18
        oidc-ca-file: "/etc/kubernetes/pki/oidc-ca.crt"

1
package main
2

3
import (
4
    "encoding/json"
5
    "fmt"
6
    "net/http"
7
    "strings"
8

9
    authv1 "k8s.io/api/authentication/v1"
10
    metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
11
)
12

13
type TokenValidator struct {
14
    // Your token validation logic
15
}
16

17
func (tv *TokenValidator) Validate(token string) (*authv1.UserInfo, error) {
18
    // Implement your custom validation
19
    // Example: validate against external IdP
20

21
    if !strings.HasPrefix(token, "Bearer ") {
22
        return nil, fmt.Errorf("invalid token format")
23
    }
24

25
    // Extract and validate token
26
    actualToken := strings.TrimPrefix(token, "Bearer ")
27

28
    // Mock validation - replace with real logic
29
    if actualToken == "valid-token-123" {
30
        return &authv1.UserInfo{
31
            Username: "john.doe",
32
            UID:      "123456",
33
            Groups:   []string{"developers", "qa"},
34
            Extra: map[string]authv1.ExtraValue{
35
                "department": []string{"engineering"},
36
            },
37
        }, nil
38
    }
39

40
    return nil, fmt.Errorf("invalid token")
41
}
42

43
func webhookHandler(w http.ResponseWriter, r *http.Request) {
44
    var tokenReview authv1.TokenReview
45

46
    if err := json.NewDecoder(r.Body).Decode(&tokenReview); err != nil {
47
        http.Error(w, err.Error(), http.StatusBadRequest)
48
        return
49
    }
50

51
    validator := &TokenValidator{}
52
    userInfo, err := validator.Validate(tokenReview.Spec.Token)
53

54
    response := &authv1.TokenReview{
55
        TypeMeta: metav1.TypeMeta{
56
            APIVersion: "authentication.k8s.io/v1",
57
            Kind:       "TokenReview",
58
        },
59
        Status: authv1.TokenReviewStatus{
60
            Authenticated: err == nil,
61
        },
62
    }
63

64
    if err == nil {
65
        response.Status.User = *userInfo
66
    } else {
67
        response.Status.Error = err.Error()
68
    }
69

70
    w.Header().Set("Content-Type", "application/json")
71
    json.NewEncoder(w).Encode(response)
72
}
73

74
func main() {
75
    http.HandleFunc("/authenticate", webhookHandler)
76

77
    fmt.Println("Webhook authenticator listening on :3000")
78
    if err := http.ListenAndServeTLS(":3000", "cert.pem", "key.pem", nil); err != nil {
79
        panic(err)
80
    }
81
}

1
#!/bin/bash
2
USER="developer-jane"
3
GROUP="developers"
4

5
# Generate private key
6
openssl genrsa -out ${USER}.key 2048
7

8
# Generate CSR
9
cat <<EOF | openssl req -new -key ${USER}.key -out ${USER}.csr -config -
10
[req]
11
distinguished_name = req_distinguished_name
12
[req_distinguished_name]
13
[v3_req]
14
keyUsage = critical, digitalSignature, keyEncipherment
15
extendedKeyUsage = clientAuth
16
EOF
17

18
# Create CSR in Kubernetes
19
cat <<EOF | kubectl apply -f -
20
apiVersion: certificates.k8s.io/v1
21
kind: CertificateSigningRequest
22
metadata:
23
  name: ${USER}
24
spec:
25
  request: $(cat ${USER}.csr | base64 | tr -d '\n')
26
  signerName: kubernetes.io/kube-apiserver-client
27
  usages:
28
  - client auth
29
EOF
30

31
# Approve CSR
32
kubectl certificate approve ${USER}
33

34
# Get certificate
35
kubectl get csr ${USER} -o jsonpath='{.status.certificate}' | base64 -d > ${USER}.crt
36

37
# Create kubeconfig
38
kubectl config set-credentials ${USER} \
39
    --client-certificate=${USER}.crt \
40
    --client-key=${USER}.key
41

42
kubectl config set-context ${USER}-context \
43
    --cluster=kubernetes \
44
    --user=${USER}

1
import jwt
2
import json
3
from flask import Flask, request, jsonify
4
from datetime import datetime, timezone
5

6
app = Flask(__name__)
7

8
# Your JWT secret or public key
9
JWT_SECRET = "your-secret-key"
10
JWT_ALGORITHM = "HS256"  # or RS256 for RSA
11

12
@app.route('/validate', methods=['POST'])
13
def validate_token():
14
    """Validate JWT token for Kubernetes webhook authentication"""
15

16
    token_review = request.get_json()
17
    token = token_review.get('spec', {}).get('token', '')
18

19
    try:
20
        # Decode and validate JWT
21
        payload = jwt.decode(
22
            token,
23
            JWT_SECRET,
24
            algorithms=[JWT_ALGORITHM],
25
            options={"verify_exp": True}
26
        )
27

28
        # Build response
29
        response = {
30
            "apiVersion": "authentication.k8s.io/v1",
31
            "kind": "TokenReview",
32
            "status": {
33
                "authenticated": True,
34
                "user": {
35
                    "username": payload.get('sub'),
36
                    "uid": payload.get('uid'),
37
                    "groups": payload.get('groups', []),
38
                    "extra": {
39
                        "email": [payload.get('email', '')],
40
                        "department": [payload.get('department', '')]
41
                    }
42
                },
43
                "audiences": payload.get('aud', [])
44
            }
45
        }
46

47
    except jwt.ExpiredSignatureError:
48
        response = {
49
            "apiVersion": "authentication.k8s.io/v1",
50
            "kind": "TokenReview",
51
            "status": {
52
                "authenticated": False,
53
                "error": "Token has expired"
54
            }
55
        }
56
    except jwt.InvalidTokenError as e:
57
        response = {
58
            "apiVersion": "authentication.k8s.io/v1",
59
            "kind": "TokenReview",
60
            "status": {
61
                "authenticated": False,
62
                "error": str(e)
63
            }
64
        }
65

66
    return jsonify(response)
67

68
if __name__ == '__main__':
69
    app.run(host='0.0.0.0', port=3000, ssl_context='adhoc')

1
apiVersion: audit.k8s.io/v1
2
kind: Policy
3
rules:
4
  # Don't log read-only requests to common resources
5
  - level: None
6
    users: ["system:kube-proxy"]
7
    verbs: ["watch"]
8
    resources:
9
    - group: ""
10
      resources: ["endpoints", "services"]
11

12
  # Don't log these read-only URLs
13
  - level: None
14
    nonResourceURLs:
15
    - /healthz*
16
    - /version
17
    - /swagger*
18

19
  # Log service account token creation at Metadata level
20
  - level: Metadata
21
    omitStages:
22
    - RequestReceived
23
    resources:
24
    - group: ""
25
      resources: ["secrets"]
26
    namespaces: ["kube-system", "kube-public"]
27

28
  # Log all changes to pods at RequestResponse level
29
  - level: RequestResponse
30
    verbs: ["create", "update", "patch", "delete"]
31
    resources:
32
    - group: ""
33
      resources: ["pods"]
34

35
  # Log all RBAC changes at RequestResponse level
36
  - level: RequestResponse
37
    resources:
38
    - group: "rbac.authorization.k8s.io"
39
      resources: ["roles", "rolebindings", "clusterroles", "clusterrolebindings"]
40

41
  # Catch all other requests at Metadata level
42
  - level: Metadata
43
    omitStages:
44
    - RequestReceived

1
apiVersion: monitoring.coreos.com/v1
2
kind: PrometheusRule
3
metadata:
4
  name: auth-monitoring
5
  namespace: monitoring
6
spec:
7
  groups:
8
  - name: authentication.rules
9
    interval: 30s
10
    rules:
11
    - alert: HighAuthenticationFailureRate
12
      expr: |
13
        rate(apiserver_authentication_attempts{result="error"}[5m]) > 0.1
14
      for: 10m
15
      labels:
16
        severity: warning
17
      annotations:
18
        summary: "High authentication failure rate detected"
19
        description: "Authentication failure rate is {{ $value }} errors per second"
20

21
    - alert: ServiceAccountTokenExpiry
22
      expr: |
23
        (kube_secret_created{type="kubernetes.io/service-account-token"}
24
        + 30 * 24 * 60 * 60) < time()
25
      for: 1h
26
      labels:
27
        severity: info
28
      annotations:
29
        summary: "Service account token older than 30 days"
30
        description: "Token {{ $labels.secret }} in namespace {{ $labels.namespace }} is old"
31

32
    - alert: UnauthorizedAPIAccess
33
      expr: |
34
        rate(apiserver_authorization_decisions{decision="deny"}[5m]) > 0.05
35
      for: 10m
36
      labels:
37
        severity: warning
38
      annotations:
39
        summary: "High rate of unauthorized API access attempts"
40
        description: "{{ $value }} denials per second detected"

1
#!/usr/bin/env python3
2
import json
3
import sys
4
from collections import defaultdict
5
from datetime import datetime
6

7
def analyze_audit_logs(log_file):
8
    """Analyze Kubernetes audit logs for security insights"""
9

10
    stats = {
11
        'total_events': 0,
12
        'auth_failures': 0,
13
        'rbac_changes': 0,
14
        'secret_access': 0,
15
        'users': defaultdict(int),
16
        'verbs': defaultdict(int),
17
        'denied_requests': []
18
    }
19

20
    with open(log_file, 'r') as f:
21
        for line in f:
22
            try:
23
                event = json.loads(line)
24
                stats['total_events'] += 1
25

26
                # Track users
27
                user = event.get('user', {}).get('username', 'unknown')
28
                stats['users'][user] += 1
29

30
                # Track verbs
31
                verb = event.get('verb', 'unknown')
32
                stats['verbs'][verb] += 1
33

34
                # Check for authentication failures
35
                if event.get('responseStatus', {}).get('code', 200) == 401:
36
                    stats['auth_failures'] += 1
37

38
                # Check for authorization failures
39
                if event.get('responseStatus', {}).get('code', 200) == 403:
40
                    stats['denied_requests'].append({
41
                        'user': user,
42
                        'verb': verb,
43
                        'resource': event.get('objectRef', {}).get('resource', 'unknown'),
44
                        'time': event.get('requestReceivedTimestamp', 'unknown')
45
                    })
46

47
                # Check for RBAC changes
48
                if event.get('objectRef', {}).get('apiGroup') == 'rbac.authorization.k8s.io':
49
                    stats['rbac_changes'] += 1
50

51
                # Check for secret access
52
                if event.get('objectRef', {}).get('resource') == 'secrets':
53
                    stats['secret_access'] += 1
54

55
            except json.JSONDecodeError:
56
                continue
57

58
    # Print analysis
59
    print("=== Audit Log Analysis ===")
60
    print(f"Total Events: {stats['total_events']}")
61
    print(f"Auth Failures: {stats['auth_failures']}")
62
    print(f"RBAC Changes: {stats['rbac_changes']}")
63
    print(f"Secret Access: {stats['secret_access']}")
64

65
    print("\n=== Top Users ===")
66
    for user, count in sorted(stats['users'].items(),
67
                              key=lambda x: x[1],
68
                              reverse=True)[:10]:
69
        print(f"{user}: {count}")
70

71
    print("\n=== Verb Distribution ===")
72
    for verb, count in sorted(stats['verbs'].items(),
73
                              key=lambda x: x[1],
74
                              reverse=True):
75
        print(f"{verb}: {count}")
76

77
    if stats['denied_requests']:
78
        print("\n=== Recent Denied Requests ===")
79
        for req in stats['denied_requests'][:10]:
80
            print(f"User: {req['user']}, Verb: {req['verb']}, "
81
                  f"Resource: {req['resource']}, Time: {req['time']}")
82

83
if __name__ == "__main__":
84
    if len(sys.argv) != 2:
85
        print("Usage: python analyze-audit-logs.py <audit-log-file>")
86
        sys.exit(1)
87

88
    analyze_audit_logs(sys.argv[1])

1
#!/bin/bash
2
echo "=== Authentication Debugging Tool ==="
3

4
# 1. Check current context and user
5
echo -e "\n1. Current Context:"
6
kubectl config current-context
7
kubectl config view --minify -o jsonpath='{.users[0].name}'
8

9
# 2. Test authentication
10
echo -e "\n2. Testing Authentication:"
11
kubectl auth whoami
12

13
# 3. Check if token is valid
14
echo -e "\n3. Token Validation:"
15
TOKEN=$(kubectl config view --raw -o jsonpath='{.users[0].user.token}')
16
if [ ! -z "$TOKEN" ]; then
17
    kubectl create -f - <<EOF
18
apiVersion: authentication.k8s.io/v1
19
kind: TokenReview
20
spec:
21
  token: $TOKEN
22
EOF
23
fi
24

25
# 4. Check RBAC permissions
26
echo -e "\n4. Can-I Checks:"
27
kubectl auth can-i --list
28

29
# 5. Test specific permissions
30
echo -e "\n5. Specific Permission Tests:"
31
resources=("pods" "services" "deployments" "secrets" "configmaps")
32
verbs=("get" "list" "create" "update" "delete")
33

34
for resource in "${resources[@]}"; do
35
    for verb in "${verbs[@]}"; do
36
        if kubectl auth can-i $verb $resource &>/dev/null; then
37
            echo "✓ Can $verb $resource"
38
        else
39
            echo "✗ Cannot $verb $resource"
40
        fi
41
    done
42
    echo
43
done
44

45
# 6. Check service account permissions
46
echo -e "\n6. Service Account Permissions:"
47
SA_NAME=${1:-default}
48
SA_NAMESPACE=${2:-default}
49

50
kubectl auth can-i --list --as=system:serviceaccount:$SA_NAMESPACE:$SA_NAME