# Example 3: PostgreSQL Password Rotation with Previous Version # # Use case: PostgreSQL user password that rotates every 90 days # # Characteristics: # - Password rotates every 90 days # - PostgreSQL REQUIRES old password to authenticate before changing # - keepPreviousVersion creates postgres-secret-previous # - CronJob uses old password to authenticate, sets new password # - After 1 hour grace period, old secret is cleaned up --- apiVersion: v1 kind: Namespace metadata: name: database --- # ManagedSecret with previous version support apiVersion: secrets.c5ai.ch/v1alpha1 kind: ManagedSecret metadata: name: postgres-app-user namespace: database spec: vault: address: "http://openbao.openbao.svc.cluster.local:8200" authMethod: kubernetes role: managedsecret-operator kvVersion: v2 mount: secret path: postgres/app-user fields: - name: username type: static value: "app_user" - name: password type: generated generator: type: password length: 40 minDigits: 8 minSymbols: 8 minLowercase: 8 minUppercase: 8 symbolCharacters: "!@#$%^&*()" allowRepeat: false - name: host type: static value: "postgres.database.svc.cluster.local" - name: port type: static value: "5432" - name: database type: static value: "app_db" destination: name: postgres-secret type: Opaque # Keep previous version for authentication during rotation keepPreviousVersion: true previousVersionTTL: 1h # Clean up after 1 hour rotation: enabled: true schedule: 2160h # 90 days rotateGeneratedOnly: true --- # ServiceAccount for password rotation job apiVersion: v1 kind: ServiceAccount metadata: name: postgres-rotator namespace: database --- # Role to allow reading secrets apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: name: postgres-rotator namespace: database rules: - apiGroups: [""] resources: ["secrets"] verbs: ["get", "list"] resourceNames: ["postgres-secret", "postgres-secret-previous"] --- # RoleBinding apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: name: postgres-rotator namespace: database roleRef: apiGroup: rbac.authorization.k8s.io kind: Role name: postgres-rotator subjects: - kind: ServiceAccount name: postgres-rotator namespace: database --- # CronJob to detect rotation and update PostgreSQL password apiVersion: batch/v1 kind: CronJob metadata: name: postgres-password-rotator namespace: database spec: # Run every 5 minutes to check for rotation schedule: "*/5 * * * *" successfulJobsHistoryLimit: 3 failedJobsHistoryLimit: 3 jobTemplate: spec: backoffLimit: 3 template: metadata: annotations: # Reloader can also trigger this job when secret changes reloader.stakater.com/auto: "true" spec: serviceAccountName: postgres-rotator restartPolicy: OnFailure containers: - name: rotate-password image: postgres:15-alpine env: # Current credentials - name: PGUSER valueFrom: secretKeyRef: name: postgres-secret key: username - name: NEW_PASSWORD valueFrom: secretKeyRef: name: postgres-secret key: password - name: PGHOST valueFrom: secretKeyRef: name: postgres-secret key: host - name: PGPORT valueFrom: secretKeyRef: name: postgres-secret key: port - name: PGDATABASE valueFrom: secretKeyRef: name: postgres-secret key: database # Previous credentials (for authentication) - name: OLD_PASSWORD valueFrom: secretKeyRef: name: postgres-secret-previous key: password optional: true # Might not exist if no rotation yet command: - sh - -c - | set -e echo "==========================================" echo "PostgreSQL Password Rotation Check" echo "==========================================" echo "User: $PGUSER" echo "Host: $PGHOST:$PGPORT" echo "Database: $PGDATABASE" echo "" # Check if previous password exists (rotation happened) if [ -z "$OLD_PASSWORD" ]; then echo "No previous password found - no rotation needed" exit 0 fi # Check if passwords are different if [ "$OLD_PASSWORD" = "$NEW_PASSWORD" ]; then echo "Passwords are identical - no rotation needed" exit 0 fi echo "Rotation detected! Old and new passwords differ." echo "" # Test if current password in PostgreSQL is the old one echo "Testing authentication with OLD password..." if PGPASSWORD=$OLD_PASSWORD psql -c "SELECT 1;" > /dev/null 2>&1; then echo "✓ Old password still active in PostgreSQL" echo "" echo "Updating to NEW password..." # Authenticate with OLD password, set NEW password if PGPASSWORD=$OLD_PASSWORD psql -c "ALTER USER $PGUSER PASSWORD '$NEW_PASSWORD';"; then echo "✓ Password updated successfully!" echo "" # Verify new password works echo "Verifying NEW password..." if PGPASSWORD=$NEW_PASSWORD psql -c "SELECT 1;" > /dev/null 2>&1; then echo "✓ New password verified and working!" echo "" echo "==========================================" echo "Rotation completed successfully" echo "==========================================" exit 0 else echo "✗ ERROR: New password doesn't work!" exit 1 fi else echo "✗ ERROR: Failed to update password" exit 1 fi else echo "Old password doesn't work - checking if new password already active..." if PGPASSWORD=$NEW_PASSWORD psql -c "SELECT 1;" > /dev/null 2>&1; then echo "✓ New password is already active in PostgreSQL" echo "Rotation was already completed" exit 0 else echo "✗ ERROR: Neither old nor new password works!" echo "Manual intervention required" exit 1 fi fi --- # Example application Deployment using the credentials apiVersion: apps/v1 kind: Deployment metadata: name: app namespace: database spec: replicas: 2 selector: matchLabels: app: postgres-client template: metadata: labels: app: postgres-client annotations: # Reloader restarts app when credentials change reloader.stakater.com/auto: "true" spec: containers: - name: app image: postgres:15-alpine env: - name: PGUSER valueFrom: secretKeyRef: name: postgres-secret key: username - name: PGPASSWORD valueFrom: secretKeyRef: name: postgres-secret key: password - name: PGHOST valueFrom: secretKeyRef: name: postgres-secret key: host - name: PGPORT valueFrom: secretKeyRef: name: postgres-secret key: port - name: PGDATABASE valueFrom: secretKeyRef: name: postgres-secret key: database command: - sh - -c - | # Your application code here while true; do psql -c "SELECT NOW();" || echo "Connection failed" sleep 30 done --- # How the rotation flow works: # # Day 0: Initial setup # - ManagedSecret creates postgres-secret with generated password # - No postgres-secret-previous exists yet # - CronJob runs every 5 minutes, finds no previous password, does nothing # # Day 90: Rotation triggered # 1. ManagedSecret operator: # - Generates NEW password # - Updates Vault with new password # - Creates postgres-secret-previous with OLD password # - Updates postgres-secret with NEW password # # 2. Within 5 minutes, CronJob runs: # - Detects postgres-secret-previous exists # - Compares OLD_PASSWORD vs NEW_PASSWORD (different!) # - Authenticates to PostgreSQL with OLD_PASSWORD # - Runs: ALTER USER app_user PASSWORD 'new_password' # - Verifies new password works # - Job succeeds # # 3. After 1 hour (previousVersionTTL): # - ManagedSecret operator deletes postgres-secret-previous # - CronJob future runs find no previous password, do nothing # # 4. Application pods: # - Reloader detects postgres-secret change # - Triggers rolling restart # - Pods reconnect with NEW password # - ~1 minute downtime per pod during restart --- # Manual rotation testing: # # # Check current state # kubectl get secret postgres-secret -n database -o yaml # kubectl get secret postgres-secret-previous -n database -o yaml # Might not exist # # # Force rotation # kubectl annotate managedsecret postgres-app-user -n database reconcile="$(date +%s)" --overwrite # # # Watch the rotation # kubectl get secrets -n database -w # # # Check CronJob executes # kubectl get jobs -n database -w # # # View rotation logs # kubectl logs -n database -l job-name=postgres-password-rotator-XXXXX # # # Verify password in PostgreSQL matches new secret # NEW_PASS=$(kubectl get secret postgres-secret -n database -o jsonpath='{.data.password}' | base64 -d) # kubectl exec -it -n database deployment/app -- psql -c "SELECT 1;" --- # Troubleshooting: # # If rotation fails: # 1. Check CronJob logs for errors # 2. Verify PostgreSQL is accessible from CronJob pod # 3. Check if old password still works in PostgreSQL # 4. Manually update if needed: # kubectl exec -it postgres-0 -n database -- psql -U postgres # ALTER USER app_user PASSWORD 'password_from_secret';