# Example 4: MinIO Admin Password Rotation with Previous Version # # Use case: MinIO admin credentials that rotate every 60 days # # Characteristics: # - Password rotates every 60 days # - MinIO Client (mc) needs old credentials to change password # - keepPreviousVersion creates minio-admin-secret-previous # - CronJob uses old credentials to authenticate, sets new password # - After 2 hour grace period, old secret is cleaned up --- apiVersion: v1 kind: Namespace metadata: name: minio --- # ManagedSecret for MinIO admin credentials apiVersion: secrets.c5ai.ch/v1alpha1 kind: ManagedSecret metadata: name: minio-admin namespace: minio spec: vault: address: "http://openbao.openbao.svc.cluster.local:8200" authMethod: kubernetes role: managedsecret-operator kvVersion: v2 mount: secret path: minio/admin fields: - name: username type: static value: "admin" - name: password type: generated generator: type: password length: 32 minDigits: 5 minSymbols: 5 minLowercase: 5 minUppercase: 5 symbolCharacters: "!@#$%^&*" allowRepeat: false - name: endpoint type: static value: "http://minio.minio.svc.cluster.local:9000" destination: name: minio-admin-secret type: Opaque # Keep previous version for authentication during rotation keepPreviousVersion: true previousVersionTTL: 2h # Longer grace period for MinIO rotation: enabled: true schedule: 1440h # 60 days rotateGeneratedOnly: true --- # ServiceAccount for password rotation job apiVersion: v1 kind: ServiceAccount metadata: name: minio-rotator namespace: minio --- # Role to allow reading secrets apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: name: minio-rotator namespace: minio rules: - apiGroups: [""] resources: ["secrets"] verbs: ["get", "list"] resourceNames: ["minio-admin-secret", "minio-admin-secret-previous"] --- # RoleBinding apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: name: minio-rotator namespace: minio roleRef: apiGroup: rbac.authorization.k8s.io kind: Role name: minio-rotator subjects: - kind: ServiceAccount name: minio-rotator namespace: minio --- # CronJob to detect rotation and update MinIO password apiVersion: batch/v1 kind: CronJob metadata: name: minio-password-rotator namespace: minio spec: # Run every 10 minutes to check for rotation schedule: "*/10 * * * *" successfulJobsHistoryLimit: 3 failedJobsHistoryLimit: 3 jobTemplate: spec: backoffLimit: 3 template: metadata: annotations: reloader.stakater.com/auto: "true" spec: serviceAccountName: minio-rotator restartPolicy: OnFailure containers: - name: rotate-password image: minio/mc:latest env: # Current credentials - name: MC_HOST_myminio value: "" # Will be set in script - name: USERNAME valueFrom: secretKeyRef: name: minio-admin-secret key: username - name: NEW_PASSWORD valueFrom: secretKeyRef: name: minio-admin-secret key: password - name: ENDPOINT valueFrom: secretKeyRef: name: minio-admin-secret key: endpoint # Previous credentials (for authentication) - name: OLD_USERNAME valueFrom: secretKeyRef: name: minio-admin-secret-previous key: username optional: true - name: OLD_PASSWORD valueFrom: secretKeyRef: name: minio-admin-secret-previous key: password optional: true command: - sh - -c - | set -e echo "==========================================" echo "MinIO Password Rotation Check" echo "==========================================" echo "User: $USERNAME" echo "Endpoint: $ENDPOINT" echo "" # Check if previous password exists (rotation happened) if [ -z "$OLD_PASSWORD" ]; then echo "No previous password found - no rotation needed" exit 0 fi # Check if passwords are different if [ "$OLD_PASSWORD" = "$NEW_PASSWORD" ]; then echo "Passwords are identical - no rotation needed" exit 0 fi echo "Rotation detected! Old and new passwords differ." echo "" # Test if current password in MinIO is the old one echo "Testing authentication with OLD credentials..." if mc alias set oldminio $ENDPOINT $OLD_USERNAME $OLD_PASSWORD > /dev/null 2>&1; then echo "✓ Old credentials still active in MinIO" echo "" # Test admin access if mc admin info oldminio > /dev/null 2>&1; then echo "✓ Admin access confirmed with old credentials" echo "" echo "Updating to NEW password..." # Change password using old credentials if mc admin user password oldminio $USERNAME $NEW_PASSWORD; then echo "✓ Password updated successfully!" echo "" # Verify new password works echo "Verifying NEW password..." if mc alias set newminio $ENDPOINT $USERNAME $NEW_PASSWORD > /dev/null 2>&1; then if mc admin info newminio > /dev/null 2>&1; then echo "✓ New password verified and working!" echo "" echo "==========================================" echo "Rotation completed successfully" echo "==========================================" exit 0 else echo "✗ ERROR: New credentials don't have admin access!" exit 1 fi else echo "✗ ERROR: New password doesn't work!" exit 1 fi else echo "✗ ERROR: Failed to update password" exit 1 fi else echo "✗ ERROR: No admin access with old credentials" exit 1 fi else echo "Old credentials don't work - checking if new credentials already active..." if mc alias set newminio $ENDPOINT $USERNAME $NEW_PASSWORD > /dev/null 2>&1; then if mc admin info newminio > /dev/null 2>&1; then echo "✓ New credentials are already active in MinIO" echo "Rotation was already completed" exit 0 else echo "✗ ERROR: New credentials exist but no admin access" exit 1 fi else echo "✗ ERROR: Neither old nor new credentials work!" echo "Manual intervention required" exit 1 fi fi --- # Example: MinIO StatefulSet (simplified) apiVersion: apps/v1 kind: StatefulSet metadata: name: minio namespace: minio spec: serviceName: minio replicas: 1 selector: matchLabels: app: minio template: metadata: labels: app: minio annotations: # Note: MinIO doesn't auto-reload credentials from env vars # You may need to restart pods manually or use a sidecar reloader.stakater.com/auto: "false" spec: containers: - name: minio image: minio/minio:latest args: - server - /data - --console-address - ":9001" env: - name: MINIO_ROOT_USER valueFrom: secretKeyRef: name: minio-admin-secret key: username - name: MINIO_ROOT_PASSWORD valueFrom: secretKeyRef: name: minio-admin-secret key: password ports: - containerPort: 9000 name: api - containerPort: 9001 name: console volumeMounts: - name: data mountPath: /data volumeClaimTemplates: - metadata: name: data spec: accessModes: ["ReadWriteOnce"] resources: requests: storage: 10Gi --- # Service for MinIO apiVersion: v1 kind: Service metadata: name: minio namespace: minio spec: selector: app: minio ports: - port: 9000 targetPort: 9000 name: api - port: 9001 targetPort: 9001 name: console --- # How the rotation flow works: # # Day 0: Initial setup # - ManagedSecret creates minio-admin-secret with generated password # - MinIO starts with these credentials # - No minio-admin-secret-previous exists yet # - CronJob runs every 10 minutes, finds no previous password, does nothing # # Day 60: Rotation triggered # 1. ManagedSecret operator: # - Generates NEW password # - Updates Vault with new password # - Creates minio-admin-secret-previous with OLD password # - Updates minio-admin-secret with NEW password # # 2. Within 10 minutes, CronJob runs: # - Detects minio-admin-secret-previous exists # - Compares OLD_PASSWORD vs NEW_PASSWORD (different!) # - Authenticates to MinIO with OLD credentials # - Runs: mc admin user password myminio admin new_password # - Verifies new password works # - Job succeeds # # 3. After 2 hours (previousVersionTTL): # - ManagedSecret operator deletes minio-admin-secret-previous # - CronJob future runs find no previous password, do nothing # # 4. MinIO pods: # - IMPORTANT: MinIO does NOT reload credentials from env vars # - You need to manually restart MinIO pods after rotation # - Or use a sidecar to monitor and trigger restart # - Consider NOT using reloader.stakater.com/auto for MinIO --- # Alternative: Sidecar to handle MinIO restart # Add this to the MinIO StatefulSet if you want automatic restarts --- # sidecar: # - name: credential-watcher # image: bitnami/kubectl:latest # command: # - sh # - -c # - | # LAST_VERSION="" # while true; do # CURRENT_VERSION=$(kubectl get secret minio-admin-secret -n minio -o jsonpath='{.metadata.resourceVersion}') # if [ -n "$LAST_VERSION" ] && [ "$LAST_VERSION" != "$CURRENT_VERSION" ]; then # echo "Credentials changed! Signaling MinIO to restart..." # # Send SIGTERM to MinIO process to trigger graceful shutdown # killall minio # fi # LAST_VERSION=$CURRENT_VERSION # sleep 30 # done --- # Manual rotation testing: # # # Check current state # kubectl get secret minio-admin-secret -n minio -o yaml # kubectl get secret minio-admin-secret-previous -n minio -o yaml # # # Force rotation # kubectl annotate managedsecret minio-admin -n minio reconcile="$(date +%s)" --overwrite # # # Watch the rotation # kubectl get secrets -n minio -w # # # Check CronJob executes # kubectl get jobs -n minio -w # # # View rotation logs # kubectl logs -n minio -l job-name=minio-password-rotator-XXXXX # # # Test MinIO access with new credentials # NEW_USER=$(kubectl get secret minio-admin-secret -n minio -o jsonpath='{.data.username}' | base64 -d) # NEW_PASS=$(kubectl get secret minio-admin-secret -n minio -o jsonpath='{.data.password}' | base64 -d) # mc alias set testminio http://minio.minio.svc.cluster.local:9000 $NEW_USER $NEW_PASS # mc admin info testminio --- # Important Notes: # # 1. MinIO credential reload: # - Unlike PostgreSQL, MinIO does NOT automatically reload credentials # - After rotation, you must restart MinIO pods # - Consider using a sidecar or manual restart workflow # # 2. High Availability: # - For HA MinIO clusters, all nodes share the same root credentials # - Rotation updates credentials cluster-wide # - Rolling restart required for all nodes # # 3. Grace Period: # - 2 hour previousVersionTTL allows time for manual intervention # - Adjust based on your operational procedures # # 4. Monitoring: # - Monitor CronJob success/failure # - Alert on rotation failures # - Track when MinIO pods were last restarted