admin
1bc8aadb85
- Streamlined README focused on quick start - Complete examples for all major use cases - Decision tree for choosing right pattern - Comprehensive troubleshooting guide
369 lines
11 KiB
YAML
369 lines
11 KiB
YAML
# Example 3: PostgreSQL Password Rotation with Previous Version
|
|
#
|
|
# Use case: PostgreSQL user password that rotates every 90 days
|
|
#
|
|
# Characteristics:
|
|
# - Password rotates every 90 days
|
|
# - PostgreSQL REQUIRES old password to authenticate before changing
|
|
# - keepPreviousVersion creates postgres-secret-previous
|
|
# - CronJob uses old password to authenticate, sets new password
|
|
# - After 1 hour grace period, old secret is cleaned up
|
|
|
|
---
|
|
apiVersion: v1
|
|
kind: Namespace
|
|
metadata:
|
|
name: database
|
|
|
|
---
|
|
# ManagedSecret with previous version support
|
|
apiVersion: secrets.c5ai.ch/v1alpha1
|
|
kind: ManagedSecret
|
|
metadata:
|
|
name: postgres-app-user
|
|
namespace: database
|
|
spec:
|
|
vault:
|
|
address: "http://openbao.openbao.svc.cluster.local:8200"
|
|
authMethod: kubernetes
|
|
role: managedsecret-operator
|
|
kvVersion: v2
|
|
mount: secret
|
|
path: postgres/app-user
|
|
|
|
fields:
|
|
- name: username
|
|
type: static
|
|
value: "app_user"
|
|
|
|
- name: password
|
|
type: generated
|
|
generator:
|
|
type: password
|
|
length: 40
|
|
minDigits: 8
|
|
minSymbols: 8
|
|
minLowercase: 8
|
|
minUppercase: 8
|
|
symbolCharacters: "!@#$%^&*()"
|
|
allowRepeat: false
|
|
|
|
- name: host
|
|
type: static
|
|
value: "postgres.database.svc.cluster.local"
|
|
|
|
- name: port
|
|
type: static
|
|
value: "5432"
|
|
|
|
- name: database
|
|
type: static
|
|
value: "app_db"
|
|
|
|
destination:
|
|
name: postgres-secret
|
|
type: Opaque
|
|
# Keep previous version for authentication during rotation
|
|
keepPreviousVersion: true
|
|
previousVersionTTL: 1h # Clean up after 1 hour
|
|
|
|
rotation:
|
|
enabled: true
|
|
schedule: 2160h # 90 days
|
|
rotateGeneratedOnly: true
|
|
|
|
---
|
|
# ServiceAccount for password rotation job
|
|
apiVersion: v1
|
|
kind: ServiceAccount
|
|
metadata:
|
|
name: postgres-rotator
|
|
namespace: database
|
|
|
|
---
|
|
# Role to allow reading secrets
|
|
apiVersion: rbac.authorization.k8s.io/v1
|
|
kind: Role
|
|
metadata:
|
|
name: postgres-rotator
|
|
namespace: database
|
|
rules:
|
|
- apiGroups: [""]
|
|
resources: ["secrets"]
|
|
verbs: ["get", "list"]
|
|
resourceNames: ["postgres-secret", "postgres-secret-previous"]
|
|
|
|
---
|
|
# RoleBinding
|
|
apiVersion: rbac.authorization.k8s.io/v1
|
|
kind: RoleBinding
|
|
metadata:
|
|
name: postgres-rotator
|
|
namespace: database
|
|
roleRef:
|
|
apiGroup: rbac.authorization.k8s.io
|
|
kind: Role
|
|
name: postgres-rotator
|
|
subjects:
|
|
- kind: ServiceAccount
|
|
name: postgres-rotator
|
|
namespace: database
|
|
|
|
---
|
|
# CronJob to detect rotation and update PostgreSQL password
|
|
apiVersion: batch/v1
|
|
kind: CronJob
|
|
metadata:
|
|
name: postgres-password-rotator
|
|
namespace: database
|
|
spec:
|
|
# Run every 5 minutes to check for rotation
|
|
schedule: "*/5 * * * *"
|
|
successfulJobsHistoryLimit: 3
|
|
failedJobsHistoryLimit: 3
|
|
jobTemplate:
|
|
spec:
|
|
backoffLimit: 3
|
|
template:
|
|
metadata:
|
|
annotations:
|
|
# Reloader can also trigger this job when secret changes
|
|
reloader.stakater.com/auto: "true"
|
|
spec:
|
|
serviceAccountName: postgres-rotator
|
|
restartPolicy: OnFailure
|
|
containers:
|
|
- name: rotate-password
|
|
image: postgres:15-alpine
|
|
env:
|
|
# Current credentials
|
|
- name: PGUSER
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: postgres-secret
|
|
key: username
|
|
- name: NEW_PASSWORD
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: postgres-secret
|
|
key: password
|
|
- name: PGHOST
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: postgres-secret
|
|
key: host
|
|
- name: PGPORT
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: postgres-secret
|
|
key: port
|
|
- name: PGDATABASE
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: postgres-secret
|
|
key: database
|
|
|
|
# Previous credentials (for authentication)
|
|
- name: OLD_PASSWORD
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: postgres-secret-previous
|
|
key: password
|
|
optional: true # Might not exist if no rotation yet
|
|
|
|
command:
|
|
- sh
|
|
- -c
|
|
- |
|
|
set -e
|
|
|
|
echo "=========================================="
|
|
echo "PostgreSQL Password Rotation Check"
|
|
echo "=========================================="
|
|
echo "User: $PGUSER"
|
|
echo "Host: $PGHOST:$PGPORT"
|
|
echo "Database: $PGDATABASE"
|
|
echo ""
|
|
|
|
# Check if previous password exists (rotation happened)
|
|
if [ -z "$OLD_PASSWORD" ]; then
|
|
echo "No previous password found - no rotation needed"
|
|
exit 0
|
|
fi
|
|
|
|
# Check if passwords are different
|
|
if [ "$OLD_PASSWORD" = "$NEW_PASSWORD" ]; then
|
|
echo "Passwords are identical - no rotation needed"
|
|
exit 0
|
|
fi
|
|
|
|
echo "Rotation detected! Old and new passwords differ."
|
|
echo ""
|
|
|
|
# Test if current password in PostgreSQL is the old one
|
|
echo "Testing authentication with OLD password..."
|
|
if PGPASSWORD=$OLD_PASSWORD psql -c "SELECT 1;" > /dev/null 2>&1; then
|
|
echo "✓ Old password still active in PostgreSQL"
|
|
echo ""
|
|
echo "Updating to NEW password..."
|
|
|
|
# Authenticate with OLD password, set NEW password
|
|
if PGPASSWORD=$OLD_PASSWORD psql -c "ALTER USER $PGUSER PASSWORD '$NEW_PASSWORD';"; then
|
|
echo "✓ Password updated successfully!"
|
|
echo ""
|
|
|
|
# Verify new password works
|
|
echo "Verifying NEW password..."
|
|
if PGPASSWORD=$NEW_PASSWORD psql -c "SELECT 1;" > /dev/null 2>&1; then
|
|
echo "✓ New password verified and working!"
|
|
echo ""
|
|
echo "=========================================="
|
|
echo "Rotation completed successfully"
|
|
echo "=========================================="
|
|
exit 0
|
|
else
|
|
echo "✗ ERROR: New password doesn't work!"
|
|
exit 1
|
|
fi
|
|
else
|
|
echo "✗ ERROR: Failed to update password"
|
|
exit 1
|
|
fi
|
|
else
|
|
echo "Old password doesn't work - checking if new password already active..."
|
|
if PGPASSWORD=$NEW_PASSWORD psql -c "SELECT 1;" > /dev/null 2>&1; then
|
|
echo "✓ New password is already active in PostgreSQL"
|
|
echo "Rotation was already completed"
|
|
exit 0
|
|
else
|
|
echo "✗ ERROR: Neither old nor new password works!"
|
|
echo "Manual intervention required"
|
|
exit 1
|
|
fi
|
|
fi
|
|
|
|
---
|
|
# Example application Deployment using the credentials
|
|
apiVersion: apps/v1
|
|
kind: Deployment
|
|
metadata:
|
|
name: app
|
|
namespace: database
|
|
spec:
|
|
replicas: 2
|
|
selector:
|
|
matchLabels:
|
|
app: postgres-client
|
|
template:
|
|
metadata:
|
|
labels:
|
|
app: postgres-client
|
|
annotations:
|
|
# Reloader restarts app when credentials change
|
|
reloader.stakater.com/auto: "true"
|
|
spec:
|
|
containers:
|
|
- name: app
|
|
image: postgres:15-alpine
|
|
env:
|
|
- name: PGUSER
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: postgres-secret
|
|
key: username
|
|
- name: PGPASSWORD
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: postgres-secret
|
|
key: password
|
|
- name: PGHOST
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: postgres-secret
|
|
key: host
|
|
- name: PGPORT
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: postgres-secret
|
|
key: port
|
|
- name: PGDATABASE
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: postgres-secret
|
|
key: database
|
|
command:
|
|
- sh
|
|
- -c
|
|
- |
|
|
# Your application code here
|
|
while true; do
|
|
psql -c "SELECT NOW();" || echo "Connection failed"
|
|
sleep 30
|
|
done
|
|
|
|
---
|
|
# How the rotation flow works:
|
|
#
|
|
# Day 0: Initial setup
|
|
# - ManagedSecret creates postgres-secret with generated password
|
|
# - No postgres-secret-previous exists yet
|
|
# - CronJob runs every 5 minutes, finds no previous password, does nothing
|
|
#
|
|
# Day 90: Rotation triggered
|
|
# 1. ManagedSecret operator:
|
|
# - Generates NEW password
|
|
# - Updates Vault with new password
|
|
# - Creates postgres-secret-previous with OLD password
|
|
# - Updates postgres-secret with NEW password
|
|
#
|
|
# 2. Within 5 minutes, CronJob runs:
|
|
# - Detects postgres-secret-previous exists
|
|
# - Compares OLD_PASSWORD vs NEW_PASSWORD (different!)
|
|
# - Authenticates to PostgreSQL with OLD_PASSWORD
|
|
# - Runs: ALTER USER app_user PASSWORD 'new_password'
|
|
# - Verifies new password works
|
|
# - Job succeeds
|
|
#
|
|
# 3. After 1 hour (previousVersionTTL):
|
|
# - ManagedSecret operator deletes postgres-secret-previous
|
|
# - CronJob future runs find no previous password, do nothing
|
|
#
|
|
# 4. Application pods:
|
|
# - Reloader detects postgres-secret change
|
|
# - Triggers rolling restart
|
|
# - Pods reconnect with NEW password
|
|
# - ~1 minute downtime per pod during restart
|
|
|
|
---
|
|
# Manual rotation testing:
|
|
#
|
|
# # Check current state
|
|
# kubectl get secret postgres-secret -n database -o yaml
|
|
# kubectl get secret postgres-secret-previous -n database -o yaml # Might not exist
|
|
#
|
|
# # Force rotation
|
|
# kubectl annotate managedsecret postgres-app-user -n database reconcile="$(date +%s)" --overwrite
|
|
#
|
|
# # Watch the rotation
|
|
# kubectl get secrets -n database -w
|
|
#
|
|
# # Check CronJob executes
|
|
# kubectl get jobs -n database -w
|
|
#
|
|
# # View rotation logs
|
|
# kubectl logs -n database -l job-name=postgres-password-rotator-XXXXX
|
|
#
|
|
# # Verify password in PostgreSQL matches new secret
|
|
# NEW_PASS=$(kubectl get secret postgres-secret -n database -o jsonpath='{.data.password}' | base64 -d)
|
|
# kubectl exec -it -n database deployment/app -- psql -c "SELECT 1;"
|
|
|
|
---
|
|
# Troubleshooting:
|
|
#
|
|
# If rotation fails:
|
|
# 1. Check CronJob logs for errors
|
|
# 2. Verify PostgreSQL is accessible from CronJob pod
|
|
# 3. Check if old password still works in PostgreSQL
|
|
# 4. Manually update if needed:
|
|
# kubectl exec -it postgres-0 -n database -- psql -U postgres
|
|
# ALTER USER app_user PASSWORD 'password_from_secret'; |