admin
1bc8aadb85
- Streamlined README focused on quick start - Complete examples for all major use cases - Decision tree for choosing right pattern - Comprehensive troubleshooting guide
430 lines
13 KiB
YAML
430 lines
13 KiB
YAML
# Example 4: MinIO Admin Password Rotation with Previous Version
|
|
#
|
|
# Use case: MinIO admin credentials that rotate every 60 days
|
|
#
|
|
# Characteristics:
|
|
# - Password rotates every 60 days
|
|
# - MinIO Client (mc) needs old credentials to change password
|
|
# - keepPreviousVersion creates minio-admin-secret-previous
|
|
# - CronJob uses old credentials to authenticate, sets new password
|
|
# - After 2 hour grace period, old secret is cleaned up
|
|
|
|
---
|
|
apiVersion: v1
|
|
kind: Namespace
|
|
metadata:
|
|
name: minio
|
|
|
|
---
|
|
# ManagedSecret for MinIO admin credentials
|
|
apiVersion: secrets.c5ai.ch/v1alpha1
|
|
kind: ManagedSecret
|
|
metadata:
|
|
name: minio-admin
|
|
namespace: minio
|
|
spec:
|
|
vault:
|
|
address: "http://openbao.openbao.svc.cluster.local:8200"
|
|
authMethod: kubernetes
|
|
role: managedsecret-operator
|
|
kvVersion: v2
|
|
mount: secret
|
|
path: minio/admin
|
|
|
|
fields:
|
|
- name: username
|
|
type: static
|
|
value: "admin"
|
|
|
|
- name: password
|
|
type: generated
|
|
generator:
|
|
type: password
|
|
length: 32
|
|
minDigits: 5
|
|
minSymbols: 5
|
|
minLowercase: 5
|
|
minUppercase: 5
|
|
symbolCharacters: "!@#$%^&*"
|
|
allowRepeat: false
|
|
|
|
- name: endpoint
|
|
type: static
|
|
value: "http://minio.minio.svc.cluster.local:9000"
|
|
|
|
destination:
|
|
name: minio-admin-secret
|
|
type: Opaque
|
|
# Keep previous version for authentication during rotation
|
|
keepPreviousVersion: true
|
|
previousVersionTTL: 2h # Longer grace period for MinIO
|
|
|
|
rotation:
|
|
enabled: true
|
|
schedule: 1440h # 60 days
|
|
rotateGeneratedOnly: true
|
|
|
|
---
|
|
# ServiceAccount for password rotation job
|
|
apiVersion: v1
|
|
kind: ServiceAccount
|
|
metadata:
|
|
name: minio-rotator
|
|
namespace: minio
|
|
|
|
---
|
|
# Role to allow reading secrets
|
|
apiVersion: rbac.authorization.k8s.io/v1
|
|
kind: Role
|
|
metadata:
|
|
name: minio-rotator
|
|
namespace: minio
|
|
rules:
|
|
- apiGroups: [""]
|
|
resources: ["secrets"]
|
|
verbs: ["get", "list"]
|
|
resourceNames: ["minio-admin-secret", "minio-admin-secret-previous"]
|
|
|
|
---
|
|
# RoleBinding
|
|
apiVersion: rbac.authorization.k8s.io/v1
|
|
kind: RoleBinding
|
|
metadata:
|
|
name: minio-rotator
|
|
namespace: minio
|
|
roleRef:
|
|
apiGroup: rbac.authorization.k8s.io
|
|
kind: Role
|
|
name: minio-rotator
|
|
subjects:
|
|
- kind: ServiceAccount
|
|
name: minio-rotator
|
|
namespace: minio
|
|
|
|
---
|
|
# CronJob to detect rotation and update MinIO password
|
|
apiVersion: batch/v1
|
|
kind: CronJob
|
|
metadata:
|
|
name: minio-password-rotator
|
|
namespace: minio
|
|
spec:
|
|
# Run every 10 minutes to check for rotation
|
|
schedule: "*/10 * * * *"
|
|
successfulJobsHistoryLimit: 3
|
|
failedJobsHistoryLimit: 3
|
|
jobTemplate:
|
|
spec:
|
|
backoffLimit: 3
|
|
template:
|
|
metadata:
|
|
annotations:
|
|
reloader.stakater.com/auto: "true"
|
|
spec:
|
|
serviceAccountName: minio-rotator
|
|
restartPolicy: OnFailure
|
|
containers:
|
|
- name: rotate-password
|
|
image: minio/mc:latest
|
|
env:
|
|
# Current credentials
|
|
- name: MC_HOST_myminio
|
|
value: "" # Will be set in script
|
|
- name: USERNAME
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: minio-admin-secret
|
|
key: username
|
|
- name: NEW_PASSWORD
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: minio-admin-secret
|
|
key: password
|
|
- name: ENDPOINT
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: minio-admin-secret
|
|
key: endpoint
|
|
|
|
# Previous credentials (for authentication)
|
|
- name: OLD_USERNAME
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: minio-admin-secret-previous
|
|
key: username
|
|
optional: true
|
|
- name: OLD_PASSWORD
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: minio-admin-secret-previous
|
|
key: password
|
|
optional: true
|
|
|
|
command:
|
|
- sh
|
|
- -c
|
|
- |
|
|
set -e
|
|
|
|
echo "=========================================="
|
|
echo "MinIO Password Rotation Check"
|
|
echo "=========================================="
|
|
echo "User: $USERNAME"
|
|
echo "Endpoint: $ENDPOINT"
|
|
echo ""
|
|
|
|
# Check if previous password exists (rotation happened)
|
|
if [ -z "$OLD_PASSWORD" ]; then
|
|
echo "No previous password found - no rotation needed"
|
|
exit 0
|
|
fi
|
|
|
|
# Check if passwords are different
|
|
if [ "$OLD_PASSWORD" = "$NEW_PASSWORD" ]; then
|
|
echo "Passwords are identical - no rotation needed"
|
|
exit 0
|
|
fi
|
|
|
|
echo "Rotation detected! Old and new passwords differ."
|
|
echo ""
|
|
|
|
# Test if current password in MinIO is the old one
|
|
echo "Testing authentication with OLD credentials..."
|
|
if mc alias set oldminio $ENDPOINT $OLD_USERNAME $OLD_PASSWORD > /dev/null 2>&1; then
|
|
echo "✓ Old credentials still active in MinIO"
|
|
echo ""
|
|
|
|
# Test admin access
|
|
if mc admin info oldminio > /dev/null 2>&1; then
|
|
echo "✓ Admin access confirmed with old credentials"
|
|
echo ""
|
|
echo "Updating to NEW password..."
|
|
|
|
# Change password using old credentials
|
|
if mc admin user password oldminio $USERNAME $NEW_PASSWORD; then
|
|
echo "✓ Password updated successfully!"
|
|
echo ""
|
|
|
|
# Verify new password works
|
|
echo "Verifying NEW password..."
|
|
if mc alias set newminio $ENDPOINT $USERNAME $NEW_PASSWORD > /dev/null 2>&1; then
|
|
if mc admin info newminio > /dev/null 2>&1; then
|
|
echo "✓ New password verified and working!"
|
|
echo ""
|
|
echo "=========================================="
|
|
echo "Rotation completed successfully"
|
|
echo "=========================================="
|
|
exit 0
|
|
else
|
|
echo "✗ ERROR: New credentials don't have admin access!"
|
|
exit 1
|
|
fi
|
|
else
|
|
echo "✗ ERROR: New password doesn't work!"
|
|
exit 1
|
|
fi
|
|
else
|
|
echo "✗ ERROR: Failed to update password"
|
|
exit 1
|
|
fi
|
|
else
|
|
echo "✗ ERROR: No admin access with old credentials"
|
|
exit 1
|
|
fi
|
|
else
|
|
echo "Old credentials don't work - checking if new credentials already active..."
|
|
if mc alias set newminio $ENDPOINT $USERNAME $NEW_PASSWORD > /dev/null 2>&1; then
|
|
if mc admin info newminio > /dev/null 2>&1; then
|
|
echo "✓ New credentials are already active in MinIO"
|
|
echo "Rotation was already completed"
|
|
exit 0
|
|
else
|
|
echo "✗ ERROR: New credentials exist but no admin access"
|
|
exit 1
|
|
fi
|
|
else
|
|
echo "✗ ERROR: Neither old nor new credentials work!"
|
|
echo "Manual intervention required"
|
|
exit 1
|
|
fi
|
|
fi
|
|
|
|
---
|
|
# Example: MinIO StatefulSet (simplified)
|
|
apiVersion: apps/v1
|
|
kind: StatefulSet
|
|
metadata:
|
|
name: minio
|
|
namespace: minio
|
|
spec:
|
|
serviceName: minio
|
|
replicas: 1
|
|
selector:
|
|
matchLabels:
|
|
app: minio
|
|
template:
|
|
metadata:
|
|
labels:
|
|
app: minio
|
|
annotations:
|
|
# Note: MinIO doesn't auto-reload credentials from env vars
|
|
# You may need to restart pods manually or use a sidecar
|
|
reloader.stakater.com/auto: "false"
|
|
spec:
|
|
containers:
|
|
- name: minio
|
|
image: minio/minio:latest
|
|
args:
|
|
- server
|
|
- /data
|
|
- --console-address
|
|
- ":9001"
|
|
env:
|
|
- name: MINIO_ROOT_USER
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: minio-admin-secret
|
|
key: username
|
|
- name: MINIO_ROOT_PASSWORD
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: minio-admin-secret
|
|
key: password
|
|
ports:
|
|
- containerPort: 9000
|
|
name: api
|
|
- containerPort: 9001
|
|
name: console
|
|
volumeMounts:
|
|
- name: data
|
|
mountPath: /data
|
|
volumeClaimTemplates:
|
|
- metadata:
|
|
name: data
|
|
spec:
|
|
accessModes: ["ReadWriteOnce"]
|
|
resources:
|
|
requests:
|
|
storage: 10Gi
|
|
|
|
---
|
|
# Service for MinIO
|
|
apiVersion: v1
|
|
kind: Service
|
|
metadata:
|
|
name: minio
|
|
namespace: minio
|
|
spec:
|
|
selector:
|
|
app: minio
|
|
ports:
|
|
- port: 9000
|
|
targetPort: 9000
|
|
name: api
|
|
- port: 9001
|
|
targetPort: 9001
|
|
name: console
|
|
|
|
---
|
|
# How the rotation flow works:
|
|
#
|
|
# Day 0: Initial setup
|
|
# - ManagedSecret creates minio-admin-secret with generated password
|
|
# - MinIO starts with these credentials
|
|
# - No minio-admin-secret-previous exists yet
|
|
# - CronJob runs every 10 minutes, finds no previous password, does nothing
|
|
#
|
|
# Day 60: Rotation triggered
|
|
# 1. ManagedSecret operator:
|
|
# - Generates NEW password
|
|
# - Updates Vault with new password
|
|
# - Creates minio-admin-secret-previous with OLD password
|
|
# - Updates minio-admin-secret with NEW password
|
|
#
|
|
# 2. Within 10 minutes, CronJob runs:
|
|
# - Detects minio-admin-secret-previous exists
|
|
# - Compares OLD_PASSWORD vs NEW_PASSWORD (different!)
|
|
# - Authenticates to MinIO with OLD credentials
|
|
# - Runs: mc admin user password myminio admin new_password
|
|
# - Verifies new password works
|
|
# - Job succeeds
|
|
#
|
|
# 3. After 2 hours (previousVersionTTL):
|
|
# - ManagedSecret operator deletes minio-admin-secret-previous
|
|
# - CronJob future runs find no previous password, do nothing
|
|
#
|
|
# 4. MinIO pods:
|
|
# - IMPORTANT: MinIO does NOT reload credentials from env vars
|
|
# - You need to manually restart MinIO pods after rotation
|
|
# - Or use a sidecar to monitor and trigger restart
|
|
# - Consider NOT using reloader.stakater.com/auto for MinIO
|
|
|
|
---
|
|
# Alternative: Sidecar to handle MinIO restart
|
|
# Add this to the MinIO StatefulSet if you want automatic restarts
|
|
---
|
|
# sidecar:
|
|
# - name: credential-watcher
|
|
# image: bitnami/kubectl:latest
|
|
# command:
|
|
# - sh
|
|
# - -c
|
|
# - |
|
|
# LAST_VERSION=""
|
|
# while true; do
|
|
# CURRENT_VERSION=$(kubectl get secret minio-admin-secret -n minio -o jsonpath='{.metadata.resourceVersion}')
|
|
# if [ -n "$LAST_VERSION" ] && [ "$LAST_VERSION" != "$CURRENT_VERSION" ]; then
|
|
# echo "Credentials changed! Signaling MinIO to restart..."
|
|
# # Send SIGTERM to MinIO process to trigger graceful shutdown
|
|
# killall minio
|
|
# fi
|
|
# LAST_VERSION=$CURRENT_VERSION
|
|
# sleep 30
|
|
# done
|
|
|
|
---
|
|
# Manual rotation testing:
|
|
#
|
|
# # Check current state
|
|
# kubectl get secret minio-admin-secret -n minio -o yaml
|
|
# kubectl get secret minio-admin-secret-previous -n minio -o yaml
|
|
#
|
|
# # Force rotation
|
|
# kubectl annotate managedsecret minio-admin -n minio reconcile="$(date +%s)" --overwrite
|
|
#
|
|
# # Watch the rotation
|
|
# kubectl get secrets -n minio -w
|
|
#
|
|
# # Check CronJob executes
|
|
# kubectl get jobs -n minio -w
|
|
#
|
|
# # View rotation logs
|
|
# kubectl logs -n minio -l job-name=minio-password-rotator-XXXXX
|
|
#
|
|
# # Test MinIO access with new credentials
|
|
# NEW_USER=$(kubectl get secret minio-admin-secret -n minio -o jsonpath='{.data.username}' | base64 -d)
|
|
# NEW_PASS=$(kubectl get secret minio-admin-secret -n minio -o jsonpath='{.data.password}' | base64 -d)
|
|
# mc alias set testminio http://minio.minio.svc.cluster.local:9000 $NEW_USER $NEW_PASS
|
|
# mc admin info testminio
|
|
|
|
---
|
|
# Important Notes:
|
|
#
|
|
# 1. MinIO credential reload:
|
|
# - Unlike PostgreSQL, MinIO does NOT automatically reload credentials
|
|
# - After rotation, you must restart MinIO pods
|
|
# - Consider using a sidecar or manual restart workflow
|
|
#
|
|
# 2. High Availability:
|
|
# - For HA MinIO clusters, all nodes share the same root credentials
|
|
# - Rotation updates credentials cluster-wide
|
|
# - Rolling restart required for all nodes
|
|
#
|
|
# 3. Grace Period:
|
|
# - 2 hour previousVersionTTL allows time for manual intervention
|
|
# - Adjust based on your operational procedures
|
|
#
|
|
# 4. Monitoring:
|
|
# - Monitor CronJob success/failure
|
|
# - Alert on rotation failures
|
|
# - Track when MinIO pods were last restarted |