Security Hardening a Kubernetes Cluster#

# kube-bench runs the CIS Kubernetes Benchmark checks
# On self-managed clusters, run as a Job:
kubectl apply -f https://raw.githubusercontent.com/aquasecurity/kube-bench/main/job.yaml

# Wait for completion and read results
kubectl wait --for=condition=complete job/kube-bench --timeout=120s
kubectl logs job/kube-bench

# On managed clusters (EKS, GKE, AKS), use the platform-specific variant:
# EKS:
kubectl apply -f https://raw.githubusercontent.com/aquasecurity/kube-bench/main/job-eks.yaml

# Trivy can scan an entire cluster
trivy k8s --report=summary cluster

# For detailed results on a specific namespace:
trivy k8s --report=all --namespace=app-production

# Or scan individual images:
trivy image --severity HIGH,CRITICAL nginx:1.25.3

# List all ClusterRoleBindings and their subjects
kubectl get clusterrolebindings -o json | jq -r '
  .items[] |
  "\(.metadata.name) -> \(.roleRef.name) : \(.subjects // [] | map("\(.kind)/\(.name)") | join(", "))"'

# Find cluster-admin bindings specifically
kubectl get clusterrolebindings -o json | jq -r '
  .items[] |
  select(.roleRef.name == "cluster-admin") |
  "\(.metadata.name): \(.subjects // [] | map("\(.kind)/\(.name)") | join(", "))"'

# Find pods running as privileged
kubectl get pods -A -o json | jq -r '
  .items[] |
  .metadata.namespace as $ns |
  .metadata.name as $pod |
  .spec.containers[] |
  select(.securityContext.privileged == true) |
  "\($ns)/\($pod)/\(.name): PRIVILEGED"'

# Pods with hostPath mounts
kubectl get pods -A -o json | jq -r '
  .items[] |
  select(.spec.volumes[]? | .hostPath != null) |
  "\(.metadata.namespace)/\(.metadata.name): hostPath volumes"'

# Pods with hostNetwork
kubectl get pods -A -o json | jq -r '
  .items[] |
  select(.spec.hostNetwork == true) |
  "\(.metadata.namespace)/\(.metadata.name): hostNetwork=true"'

# Pods with hostPID
kubectl get pods -A -o json | jq -r '
  .items[] |
  select(.spec.hostPID == true) |
  "\(.metadata.namespace)/\(.metadata.name): hostPID=true"'

# List them first (from Step 3 output)
kubectl get clusterrolebindings -o json | jq -r '
  .items[] |
  select(.roleRef.name == "cluster-admin") |
  .metadata.name'

# Delete non-essential bindings (DO NOT delete system bindings)
# System bindings to keep: system:*, kubeadm:*, cluster-admin (the built-in one)
# Review each one before deleting:
kubectl delete clusterrolebinding <binding-name>

cat <<'EOF' | kubectl apply -f -
---
# Developer role: can manage workloads but not RBAC or secrets
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
  name: developer
  namespace: app-production
rules:
- apiGroups: ["apps"]
  resources: ["deployments", "replicasets", "statefulsets"]
  verbs: ["get", "list", "watch", "create", "update", "patch"]
- apiGroups: [""]
  resources: ["pods", "services", "configmaps"]
  verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]
- apiGroups: [""]
  resources: ["pods/log"]
  verbs: ["get"]
- apiGroups: [""]
  resources: ["pods/exec"]
  verbs: ["create"]
---
# Read-only role for auditors and monitoring tools
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: readonly-all
rules:
- apiGroups: ["", "apps", "batch", "networking.k8s.io"]
  resources: ["*"]
  verbs: ["get", "list", "watch"]
---
# CI/CD deployer: can update deployments and configmaps, nothing else
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
  name: deployer
  namespace: app-production
rules:
- apiGroups: ["apps"]
  resources: ["deployments"]
  verbs: ["get", "list", "patch", "update"]
- apiGroups: [""]
  resources: ["configmaps", "secrets"]
  verbs: ["get", "list", "create", "update", "patch"]
EOF

# Find pods using the default service account
kubectl get pods -A -o json | jq -r '
  .items[] |
  select(.spec.serviceAccountName == "default" or .spec.serviceAccountName == null) |
  select(.metadata.namespace != "kube-system") |
  "\(.metadata.namespace)/\(.metadata.name)"'

# Create dedicated service accounts for each workload
kubectl create serviceaccount web-api -n app-production

# Disable auto-mounting of service account tokens on pods that do not need Kubernetes API access
cat <<'EOF' | kubectl apply -f -
apiVersion: v1
kind: ServiceAccount
metadata:
  name: web-api
  namespace: app-production
automountServiceAccountToken: false
EOF

spec:
  template:
    spec:
      serviceAccountName: web-api
      automountServiceAccountToken: false  # unless the pod needs API access

# Test developer permissions
kubectl auth can-i create deployments -n app-production --as=developer@company.com
# Should return: yes

kubectl auth can-i create clusterroles --as=developer@company.com
# Should return: no

# Test deployer permissions
kubectl auth can-i delete namespaces --as=system:serviceaccount:app-production:deploy-bot
# Should return: no

# Verify no stray cluster-admin bindings
kubectl get clusterrolebindings -o json | jq '[.items[] | select(.roleRef.name == "cluster-admin") | .metadata.name]'
# Only system bindings should remain

# Enforce baseline on all application namespaces
# Warn and audit on restricted to identify further hardening opportunities
for ns in app-production app-staging; do
  kubectl label namespace $ns \
    pod-security.kubernetes.io/enforce=baseline \
    pod-security.kubernetes.io/enforce-version=latest \
    pod-security.kubernetes.io/warn=restricted \
    pod-security.kubernetes.io/warn-version=latest \
    pod-security.kubernetes.io/audit=restricted \
    pod-security.kubernetes.io/audit-version=latest \
    --overwrite
done

# Monitoring and system namespaces need more permissive settings
# because components like node-exporter and promtail need host access
kubectl label namespace monitoring \
  pod-security.kubernetes.io/enforce=privileged \
  pod-security.kubernetes.io/warn=baseline \
  --overwrite

spec:
  containers:
  - name: app
    image: myapp:1.2.3
    securityContext:
      runAsNonRoot: true
      runAsUser: 1000
      runAsGroup: 1000
      allowPrivilegeEscalation: false
      readOnlyRootFilesystem: true
      capabilities:
        drop:
          - ALL
      seccompProfile:
        type: RuntimeDefault
    volumeMounts:
    - name: tmp
      mountPath: /tmp
  volumes:
  - name: tmp
    emptyDir: {}

helm repo add kyverno https://kyverno.github.io/kyverno
helm install kyverno kyverno/kyverno \
  --namespace kyverno \
  --create-namespace \
  --set replicaCount=3 \
  --set resources.requests.cpu=100m \
  --set resources.requests.memory=256Mi

# Custom policy: require resource limits on all containers
cat <<'EOF' | kubectl apply -f -
apiVersion: kyverno.io/v1
kind: ClusterPolicy
metadata:
  name: require-resource-limits
spec:
  validationFailureAction: Enforce
  background: true
  rules:
  - name: check-limits
    match:
      any:
      - resources:
          kinds:
          - Pod
          namespaces:
          - app-production
          - app-staging
    validate:
      message: "All containers must have CPU and memory limits."
      pattern:
        spec:
          containers:
          - resources:
              limits:
                memory: "?*"
                cpu: "?*"
---
# Custom policy: restrict image registries
apiVersion: kyverno.io/v1
kind: ClusterPolicy
metadata:
  name: restrict-image-registries
spec:
  validationFailureAction: Enforce
  background: true
  rules:
  - name: validate-registries
    match:
      any:
      - resources:
          kinds:
          - Pod
          namespaces:
          - app-production
    validate:
      message: "Images must come from approved registries: registry.company.com or gcr.io/my-project."
      pattern:
        spec:
          containers:
          - image: "registry.company.com/* | gcr.io/my-project/*"
EOF

# Test PSA enforcement
kubectl run priv-test --image=busybox:1.36 --restart=Never -n app-production \
  --overrides='{"spec":{"containers":[{"name":"test","image":"busybox:1.36","securityContext":{"privileged":true}}]}}'
# Expected: Error from server (Forbidden): pod security standards violation

# Test Kyverno policy: deploy without resource limits
kubectl run no-limits --image=busybox:1.36 --restart=Never -n app-production
# Expected: Error from server: admission webhook denied the request

# Test registry restriction
kubectl run bad-registry --image=docker.io/nginx:latest --restart=Never -n app-production
# Expected: Error from server: admission webhook denied the request

# Apply to every application namespace
for ns in app-production app-staging; do
cat <<EOF | kubectl apply -f -
---
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
  name: default-deny-all
  namespace: $ns
spec:
  podSelector: {}
  policyTypes:
  - Ingress
  - Egress
---
# CRITICAL: Allow DNS -- without this, service discovery breaks entirely
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
  name: allow-dns
  namespace: $ns
spec:
  podSelector: {}
  policyTypes:
  - Egress
  egress:
  - ports:
    - protocol: UDP
      port: 53
    - protocol: TCP
      port: 53
EOF
done

cat <<'EOF' | kubectl apply -f -
---
# Allow ingress controller to reach app pods
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
  name: allow-ingress-to-web
  namespace: app-production
spec:
  podSelector:
    matchLabels:
      app: web-api
  policyTypes:
  - Ingress
  ingress:
  - from:
    - namespaceSelector:
        matchLabels:
          kubernetes.io/metadata.name: ingress-system
    ports:
    - protocol: TCP
      port: 8080
---
# Allow web-api to talk to the database
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
  name: allow-web-to-db
  namespace: app-production
spec:
  podSelector:
    matchLabels:
      app: web-api
  policyTypes:
  - Egress
  egress:
  - to:
    - podSelector:
        matchLabels:
          app: postgres
    ports:
    - protocol: TCP
      port: 5432
---
# Allow Prometheus to scrape metrics from all pods
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
  name: allow-prometheus-scrape
  namespace: app-production
spec:
  podSelector: {}
  policyTypes:
  - Ingress
  ingress:
  - from:
    - namespaceSelector:
        matchLabels:
          kubernetes.io/metadata.name: monitoring
      podSelector:
        matchLabels:
          app.kubernetes.io/name: prometheus
    ports:
    - protocol: TCP
      port: 9090
    - protocol: TCP
      port: 8080
EOF

kubectl run dns-test --image=busybox:1.36 --restart=Never --rm -it -n app-production -- nslookup kubernetes.default.svc.cluster.local

# Verify policies are applied
kubectl get networkpolicy -n app-production

# Test allowed traffic: ingress to web-api should work
kubectl exec -n ingress-system deploy/ingress-nginx-controller -- \
  wget -qO- --timeout=5 http://web-api.app-production:8080/healthz

# Test blocked traffic: direct pod-to-pod across namespaces should fail
kubectl run test-block --image=busybox:1.36 --restart=Never --rm -it -n app-staging -- \
  wget -qO- --timeout=5 http://web-api.app-production:8080/healthz
# Expected: timeout (traffic blocked)

cat <<'EOF' | kubectl apply -f -
apiVersion: kyverno.io/v1
kind: ClusterPolicy
metadata:
  name: require-image-digest
spec:
  validationFailureAction: Enforce
  background: true
  rules:
  - name: require-digest
    match:
      any:
      - resources:
          kinds:
          - Pod
          namespaces:
          - app-production
    validate:
      message: "Images must use a digest (sha256) reference, not a tag."
      pattern:
        spec:
          containers:
          - image: "*@sha256:*"
EOF

kubectl get clusterpolicy restrict-image-registries -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}'
# Should return: True

# Using Cosign with Kyverno
cat <<'EOF' | kubectl apply -f -
apiVersion: kyverno.io/v1
kind: ClusterPolicy
metadata:
  name: verify-image-signature
spec:
  validationFailureAction: Enforce
  webhookTimeoutSeconds: 30
  rules:
  - name: check-signature
    match:
      any:
      - resources:
          kinds:
          - Pod
          namespaces:
          - app-production
    verifyImages:
    - imageReferences:
      - "registry.company.com/*"
      attestors:
      - entries:
        - keys:
            publicKeys: |
              -----BEGIN PUBLIC KEY-----
              YOUR_COSIGN_PUBLIC_KEY_HERE
              -----END PUBLIC KEY-----
EOF

# Try deploying with a tag (should fail)
kubectl run tag-test --image=nginx:1.25.3 --restart=Never -n app-production
# Expected: blocked by require-image-digest policy

# Try deploying from unauthorized registry (should fail)
kubectl run reg-test --image=docker.io/nginx@sha256:abc123 --restart=Never -n app-production
# Expected: blocked by restrict-image-registries policy

# Audit policy file (place on control plane nodes)
cat <<'EOF' > /etc/kubernetes/audit-policy.yaml
apiVersion: audit.k8s.io/v1
kind: Policy
rules:
# Log secret access at Metadata level (who accessed, not what)
- level: Metadata
  resources:
  - group: ""
    resources: ["secrets"]

# Log all RBAC changes at RequestResponse level
- level: RequestResponse
  resources:
  - group: "rbac.authorization.k8s.io"
    resources: ["clusterrolebindings", "rolebindings", "clusterroles", "roles"]

# Log pod exec and attach
- level: Request
  resources:
  - group: ""
    resources: ["pods/exec", "pods/attach"]

# Log authentication failures
- level: Metadata
  stages:
  - ResponseComplete
  omitStages:
  - RequestReceived

# Catch-all: log everything else at Metadata level
- level: Metadata
EOF

cat <<'EOF' | kubectl apply -f -
apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: security-alerts
  namespace: monitoring
  labels:
    release: kube-prometheus
spec:
  groups:
  - name: security
    rules:
    - alert: PodExecDetected
      expr: increase(apiserver_request_total{verb="create", resource="pods", subresource="exec"}[5m]) > 0
      labels:
        severity: warning
      annotations:
        summary: "kubectl exec detected"
        description: "Someone exec'd into a pod in the last 5 minutes. Check audit logs."

    - alert: NewClusterAdminBinding
      expr: increase(apiserver_request_total{verb="create", resource="clusterrolebindings"}[5m]) > 0
      labels:
        severity: critical
      annotations:
        summary: "New ClusterRoleBinding created"
        description: "A new ClusterRoleBinding was created. Verify it is authorized."

    - alert: FailedAuthAttempts
      expr: increase(apiserver_audit_event_total{stage="ResponseComplete",code=~"40[13]"}[5m]) > 10
      labels:
        severity: warning
      annotations:
        summary: "Multiple failed auth attempts"
        description: "More than 10 failed authentication/authorization attempts in 5 minutes."
EOF

helm repo add falcosecurity https://falcosecurity.github.io/charts
helm install falco falcosecurity/falco \
  --namespace falco \
  --create-namespace \
  --set falcosidekick.enabled=true \
  --set falcosidekick.config.slack.webhookurl="https://hooks.slack.com/services/YOUR/SLACK/WEBHOOK" \
  --set driver.kind=modern_ebpf \
  --set collectors.kubernetes.enabled=true

# Test audit logging: exec into a pod
kubectl exec -it deploy/web-api -n app-production -- whoami

# Check audit log for the exec event
# On self-managed: grep "exec" /var/log/kubernetes/audit.log | tail -5
# On managed: check cloud provider's audit log console

# Verify Falco detects the exec
kubectl logs -l app.kubernetes.io/name=falco -n falco --tail=20 | grep "exec"
# Should show a notice about a shell being spawned in a container

cat <<'EOF' > /etc/kubernetes/encryption-config.yaml
apiVersion: apiserver.config.k8s.io/v1
kind: EncryptionConfiguration
resources:
- resources:
  - secrets
  - configmaps
  providers:
  - aescbc:
      keys:
      - name: key1
        secret: $(head -c 32 /dev/urandom | base64)
  - identity: {}
EOF

kubectl get secrets -A -o json | kubectl replace -f -

# On self-managed clusters using kubeadm:
kubeadm certs check-expiration

# For any cluster, check the API server certificate:
echo | openssl s_client -connect $(kubectl config view --minify -o jsonpath='{.clusters[0].cluster.server}' | sed 's|https://||') 2>/dev/null | openssl x509 -noout -dates

# Rotate a Kubernetes secret
kubectl create secret generic db-credentials \
  --from-literal=password=$(openssl rand -base64 24) \
  -n app-production \
  --dry-run=client -o yaml | kubectl apply -f -

# Trigger a rolling restart to pick up the new secret
kubectl rollout restart deployment/web-api -n app-production

# Verify etcd encryption (self-managed only)
# Read a secret directly from etcd -- it should be encrypted
ETCDCTL_API=3 etcdctl get /registry/secrets/app-production/db-credentials \
  --endpoints=https://127.0.0.1:2379 \
  --cacert=/etc/kubernetes/pki/etcd/ca.crt \
  --cert=/etc/kubernetes/pki/etcd/server.crt \
  --key=/etc/kubernetes/pki/etcd/server.key | hexdump -C | head
# Should show encrypted bytes, not plaintext

# Check cert expiry
kubeadm certs check-expiration 2>/dev/null || echo "Managed cluster -- check via cloud console"
# All certs > 90 days from expiry