SOPS ????????? Multi-Tenant Architecture
SOPS (Secrets OPerationS) ?????????????????????????????????????????????????????????????????? secrets ?????????????????? config ???????????????????????????????????????????????? values ?????????????????????????????????????????? ???????????????????????? diff ????????? review ?????? Git ????????????????????? ????????????????????????????????????????????? Multi-Tenant Architecture ?????????????????????????????????????????????????????? tenant ?????? encryption keys ?????????????????? ??????????????????????????? decrypt secrets ????????? tenant ?????????????????????
Multi-Tenant Design ?????????????????? SOPS ???????????????????????????????????? Key-per-Tenant ??????????????? tenant ?????? encryption key ????????? ????????????????????????????????? strict isolation, Key-per-Environment ????????? key ????????? environment (dev/staging/prod) ????????????????????????????????? single-tenant multi-env, Hierarchical Keys ????????? master key + tenant keys ?????????????????? key recovery ????????? admin access
???????????????????????? SOPS ?????????????????? multi-tenant ????????? secrets ?????????????????? Git ????????????????????? code, ??????????????? tenant decrypt ???????????????????????? secrets ???????????????????????????, CI/CD pipeline ????????? service account key ????????? scope ??????????????? tenant, audit trail ???????????? Git history ????????? KMS logging
?????????????????? Key Management ?????????????????? Multi-Tenant
?????????????????? key management architecture
# === SOPS Multi-Tenant Key Management ===
# 1. Directory Structure
# secrets-repo/
# ????????? .sops.yaml # SOPS config (key mapping)
# ????????? tenants/
# ??? ????????? tenant-alpha/
# ??? ??? ????????? production/
# ??? ??? ??? ????????? db.enc.yaml
# ??? ??? ??? ????????? api-keys.enc.yaml
# ??? ??? ??? ????????? tls.enc.yaml
# ??? ??? ????????? staging/
# ??? ??? ??? ????????? db.enc.yaml
# ??? ??? ??? ????????? api-keys.enc.yaml
# ??? ??? ????????? dev/
# ??? ??? ????????? db.enc.yaml
# ??? ????????? tenant-beta/
# ??? ??? ????????? production/
# ??? ??? ????????? staging/
# ??? ????????? tenant-gamma/
# ??? ????????? production/
# ????????? shared/
# ????????? infrastructure/
# ????????? cert-manager.enc.yaml
# ????????? monitoring.enc.yaml
# 2. .sops.yaml ??? Key Mapping per Tenant
cat > .sops.yaml << 'EOF'
creation_rules:
# Tenant Alpha ??? uses dedicated AWS KMS key
- path_regex: tenants/tenant-alpha/production/.*
kms: "arn:aws:kms:ap-southeast-1:111111111:key/alpha-prod-key-id"
age: "age1alpha_admin_key"
- path_regex: tenants/tenant-alpha/staging/.*
kms: "arn:aws:kms:ap-southeast-1:111111111:key/alpha-staging-key-id"
age: "age1alpha_admin_key"
- path_regex: tenants/tenant-alpha/dev/.*
age: "age1alpha_dev_key,age1alpha_admin_key"
# Tenant Beta ??? uses GCP KMS
- path_regex: tenants/tenant-beta/production/.*
gcp_kms: "projects/tenant-beta/locations/asia-southeast1/keyRings/sops/cryptoKeys/prod"
age: "age1beta_admin_key"
- path_regex: tenants/tenant-beta/staging/.*
gcp_kms: "projects/tenant-beta/locations/asia-southeast1/keyRings/sops/cryptoKeys/staging"
# Tenant Gamma ??? uses Azure Key Vault
- path_regex: tenants/tenant-gamma/.*
azure_keyvault: "https://gamma-vault.vault.azure.net/keys/sops-key/version"
# Shared infrastructure ??? platform team only
- path_regex: shared/.*
kms: "arn:aws:kms:ap-southeast-1:111111111:key/platform-master-key"
age: "age1platform_admin_key"
# Default ??? catch-all with age
- age: "age1platform_admin_key"
EOF
# 3. Generate Keys per Tenant
# Platform admin key
age-keygen -o keys/platform-admin.key
# Public key: age1platform_admin_key
# Tenant Alpha
age-keygen -o keys/tenant-alpha-admin.key
age-keygen -o keys/tenant-alpha-dev.key
# Tenant Beta
age-keygen -o keys/tenant-beta-admin.key
echo "Key management configured"Implement Tenant Isolation
??????????????????????????? tenant isolation ?????????????????? secrets
#!/usr/bin/env python3
# tenant_isolation.py ??? SOPS Multi-Tenant Isolation
import json
import logging
import subprocess
from pathlib import Path
from typing import Dict, List
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger("isolation")
class TenantSecretsManager:
def __init__(self, base_path="tenants"):
self.base_path = Path(base_path)
self.tenants = {}
def create_tenant(self, tenant_id, environments=None):
"""Create tenant directory structure"""
if environments is None:
environments = ["production", "staging", "dev"]
tenant_path = self.base_path / tenant_id
for env in environments:
env_path = tenant_path / env
env_path.mkdir(parents=True, exist_ok=True)
logger.info(f"Created: {env_path}")
self.tenants[tenant_id] = {
"path": str(tenant_path),
"environments": environments,
"created": True,
}
return self.tenants[tenant_id]
def encrypt_secret(self, tenant_id, environment, filename, data):
"""Encrypt secret file for specific tenant"""
file_path = self.base_path / tenant_id / environment / filename
# Write plaintext temporarily
temp_path = file_path.with_suffix(".tmp")
with open(temp_path, "w") as f:
if filename.endswith(".yaml") or filename.endswith(".yml"):
import yaml
yaml.dump(data, f)
else:
json.dump(data, f, indent=2)
# Encrypt with SOPS (uses .sops.yaml rules)
result = subprocess.run(
["sops", "--encrypt", str(temp_path)],
capture_output=True, text=True
)
if result.returncode == 0:
with open(str(file_path), "w") as f:
f.write(result.stdout)
temp_path.unlink()
logger.info(f"Encrypted: {file_path}")
return True
else:
logger.error(f"Encryption failed: {result.stderr}")
return False
def verify_isolation(self, tenant_id, other_tenant_id):
"""Verify tenant cannot decrypt other tenant's secrets"""
tenant_key = f"keys/{tenant_id}-admin.key"
other_file = self.base_path / other_tenant_id / "production" / "db.enc.yaml"
result = subprocess.run(
["sops", "--decrypt", str(other_file)],
capture_output=True, text=True,
env={"SOPS_AGE_KEY_FILE": tenant_key}
)
if result.returncode != 0:
return {"isolated": True, "message": f"{tenant_id} cannot decrypt {other_tenant_id} secrets"}
else:
return {"isolated": False, "message": "ISOLATION BREACH!"}
def list_tenant_secrets(self, tenant_id):
"""List all encrypted secrets for a tenant"""
tenant_path = self.base_path / tenant_id
secrets = []
if tenant_path.exists():
for f in tenant_path.rglob("*.enc.*"):
secrets.append({
"path": str(f.relative_to(self.base_path)),
"size": f.stat().st_size,
"environment": f.parent.name,
})
return {"tenant": tenant_id, "secrets_count": len(secrets), "secrets": secrets}
manager = TenantSecretsManager()
result = manager.create_tenant("tenant-alpha", ["production", "staging", "dev"])
print("Tenant created:", json.dumps(result, indent=2))
listing = manager.list_tenant_secrets("tenant-alpha")
print("Secrets:", json.dumps(listing, indent=2))CI/CD Pipeline ????????? SOPS Multi-Tenant
??????????????? CI/CD pipeline ??????????????????????????? multi-tenant secrets
# === CI/CD Pipeline with SOPS Multi-Tenant ===
# GitHub Actions Workflow
cat > .github/workflows/deploy-tenant.yml << 'EOF'
name: Deploy Tenant
on:
push:
paths:
- 'tenants/**'
jobs:
detect-changes:
runs-on: ubuntu-latest
outputs:
tenants: }
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 2
- id: changes
run: |
CHANGED=$(git diff --name-only HEAD~1 HEAD -- tenants/ | \
cut -d'/' -f2 | sort -u | jq -R -s -c 'split("\n") | map(select(. != ""))')
echo "tenants=$CHANGED" >> $GITHUB_OUTPUT
deploy:
needs: detect-changes
runs-on: ubuntu-latest
strategy:
matrix:
tenant: }
steps:
- uses: actions/checkout@v4
- name: Install SOPS
run: |
curl -LO https://github.com/getsops/sops/releases/download/v3.9.0/sops-v3.9.0.linux.amd64
chmod +x sops-v3.9.0.linux.amd64
sudo mv sops-v3.9.0.linux.amd64 /usr/local/bin/sops
- name: Configure AWS KMS
if: contains(matrix.tenant, 'alpha')
uses: aws-actions/configure-aws-credentials@v4
with:
role-to-assume: arn:aws:iam::111111111:role/sops-}
aws-region: ap-southeast-1
- name: Decrypt Secrets
env:
SOPS_AGE_KEY: ', matrix.tenant)] }}
run: |
for f in tenants/}/production/*.enc.yaml; do
outfile=".yaml"
sops --decrypt "$f" > "$outfile"
done
- name: Deploy to Kubernetes
run: |
kubectl config use-context }-production
kubectl apply -f tenants/}/production/*.yaml -n }
- name: Cleanup
if: always()
run: |
find tenants/ -name "*.yaml" ! -name "*.enc.yaml" -delete
EOF
# Flux CD Integration (GitOps)
cat > tenants/tenant-alpha/production/kustomization.yaml << 'EOF'
apiVersion: kustomize.toolkit.fluxcd.io/v1
kind: Kustomization
metadata:
name: tenant-alpha-secrets
namespace: flux-system
spec:
interval: 5m
sourceRef:
kind: GitRepository
name: secrets-repo
path: ./tenants/tenant-alpha/production
prune: true
decryption:
provider: sops
secretRef:
name: sops-age-tenant-alpha
EOF
echo "CI/CD pipeline configured"Key Rotation ????????? Access Control
?????????????????? key rotation ?????????????????? multi-tenant
#!/usr/bin/env python3
# key_rotation.py ??? Multi-Tenant Key Rotation
import json
import logging
import subprocess
from datetime import datetime, timedelta
from pathlib import Path
from typing import Dict, List
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger("rotation")
class KeyRotationManager:
def __init__(self):
self.rotation_policy = {
"production": 90, # days
"staging": 180,
"dev": 365,
}
def check_key_age(self, tenant_id):
"""Check if keys need rotation"""
# In production, read from key metadata/database
key_info = {
"tenant": tenant_id,
"keys": [
{
"key_id": f"{tenant_id}-prod-key",
"environment": "production",
"created": "2024-01-15",
"age_days": 150,
"max_age_days": 90,
"needs_rotation": True,
},
{
"key_id": f"{tenant_id}-staging-key",
"environment": "staging",
"created": "2024-03-01",
"age_days": 105,
"max_age_days": 180,
"needs_rotation": False,
},
],
}
return key_info
def rotate_tenant_keys(self, tenant_id, environment):
"""Rotate encryption keys for a tenant"""
steps = [
f"1. Generate new age key for {tenant_id}/{environment}",
"2. Update .sops.yaml with new public key",
"3. Re-encrypt all files: sops updatekeys tenants/{tenant_id}/{environment}/*.enc.yaml",
"4. Commit and push changes",
"5. Update CI/CD secrets with new private key",
"6. Verify decryption works with new key",
"7. Revoke old key after grace period (7 days)",
]
return {
"tenant": tenant_id,
"environment": environment,
"status": "rotation_plan",
"steps": steps,
"commands": [
f"age-keygen -o keys/{tenant_id}-{environment}-new.key",
f"# Update .sops.yaml with new public key",
f"find tenants/{tenant_id}/{environment} -name '*.enc.*' -exec sops updatekeys {{}} \\;",
f"git add -A && git commit -m 'Rotate keys for {tenant_id}/{environment}'",
],
}
def access_control_matrix(self):
"""Define who can access which tenant secrets"""
return {
"roles": {
"platform_admin": {
"description": "Platform team ??? access all tenants",
"access": ["all_tenants", "shared_infrastructure"],
"key_type": "master_key + all tenant keys",
},
"tenant_admin": {
"description": "Tenant admin ??? access own tenant only",
"access": ["own_tenant_all_environments"],
"key_type": "tenant-specific key",
},
"tenant_developer": {
"description": "Developer ??? access dev/staging only",
"access": ["own_tenant_dev", "own_tenant_staging"],
"key_type": "tenant dev key (no prod access)",
},
"ci_cd_service": {
"description": "CI/CD pipeline ??? deploy specific tenant/env",
"access": ["specific_tenant_specific_environment"],
"key_type": "scoped service account key",
},
},
}
rotator = KeyRotationManager()
check = rotator.check_key_age("tenant-alpha")
print("Key Status:", json.dumps(check, indent=2))
plan = rotator.rotate_tenant_keys("tenant-alpha", "production")
print("\nRotation Plan:")
for step in plan["steps"]:
print(f" {step}")
acl = rotator.access_control_matrix()
print("\nAccess Roles:")
for role, info in acl["roles"].items():
print(f" {role}: {info['description']}")Monitoring ????????? Audit
Monitor secrets usage ????????? audit trail
# === SOPS Audit & Monitoring ===
# 1. AWS KMS Audit (CloudTrail)
# Every SOPS decrypt/encrypt calls KMS
# CloudTrail logs: Decrypt, Encrypt, GenerateDataKey events
# Filter: eventSource = kms.amazonaws.com
# Query CloudTrail for SOPS operations:
cat > audit_query.sh << 'BASH'
#!/bin/bash
# Find all KMS decrypt operations in last 24 hours
aws cloudtrail lookup-events \
--lookup-attributes AttributeKey=EventName,AttributeValue=Decrypt \
--start-time $(date -u -d '24 hours ago' +%Y-%m-%dT%H:%M:%S) \
--query 'Events[*].{Time:EventTime,User:Username,Key:Resources[0].ResourceName}' \
--output table
# Count operations per user
aws cloudtrail lookup-events \
--lookup-attributes AttributeKey=EventName,AttributeValue=Decrypt \
--start-time $(date -u -d '7 days ago' +%Y-%m-%dT%H:%M:%S) \
--query 'Events[*].Username' \
--output text | sort | uniq -c | sort -rn
BASH
# 2. Git-based Audit Trail
# Every secret change is a Git commit
# Who changed what, when, and why (commit message)
cat > audit_git.sh << 'BASH'
#!/bin/bash
# Audit secret changes per tenant
TENANT=
echo "=== Secret Changes for $TENANT ==="
git log --oneline --since="30 days ago" -- "tenants/$TENANT/" | head -20
echo -e "\n=== Changed Files ==="
git log --name-only --since="30 days ago" -- "tenants/$TENANT/" | grep "\.enc\." | sort | uniq -c | sort -rn
echo -e "\n=== Contributors ==="
git log --format='%an' --since="30 days ago" -- "tenants/$TENANT/" | sort | uniq -c | sort -rn
BASH
chmod +x audit_query.sh audit_git.sh
# 3. Pre-commit Hook ??? Prevent Plaintext Secrets
cat > .pre-commit-config.yaml << 'EOF'
repos:
- repo: local
hooks:
- id: check-sops-encryption
name: Verify SOPS encryption
entry: bash -c '
for f in $(git diff --cached --name-only | grep -E "tenants/.*\.(yaml|json)$" | grep -v "\.enc\."); do
echo "ERROR: Unencrypted secret file: $f"
echo "Use: sops --encrypt $f > .enc.yaml"
exit 1
done
for f in $(git diff --cached --name-only | grep "\.enc\."); do
if ! grep -q "sops" "$f" 2>/dev/null; then
echo "ERROR: $f appears unencrypted!"
exit 1
fi
done
'
language: system
pass_filenames: false
- id: detect-secrets
name: Detect hardcoded secrets
entry: bash -c '
PATTERNS="password|secret|api_key|token|private_key"
for f in $(git diff --cached --name-only | grep -v "\.enc\."); do
if grep -iP "($PATTERNS)\s*[:=]\s*[\"'\'']\S+" "$f" 2>/dev/null; then
echo "WARNING: Possible plaintext secret in $f"
fi
done
'
language: system
pass_filenames: false
EOF
echo "Audit and monitoring configured"FAQ ??????????????????????????????????????????
Q: SOPS Multi-Tenant ????????? HashiCorp Vault Multi-Tenant ??????????????????????????????????
A: SOPS Multi-Tenant ?????????????????????????????? ?????????????????? GitOps workflow ????????????????????????, secrets ?????????????????????????????????????????? (config, API keys), ????????????????????? audit trail ???????????? Git, ??????????????????????????????????????? infrastructure ??????????????? Vault Multi-Tenant ?????????????????????????????? ????????????????????? dynamic secrets (auto-generated, auto-rotated), access control ????????????????????? (policies per path), secrets ?????????????????????????????????, ????????????????????? centralized secrets management ?????????????????? Kubernetes ????????? SOPS + Flux (decrypt at deploy) ???????????? External Secrets Operator + Vault (fetch at runtime) ??????????????????????????????????????? SOPS ?????????????????? static config, Vault ?????????????????? dynamic secrets
Q: ????????? tenant ????????????????????? key ??????????????????????????? ????????????????????????????????????????
A: ?????? 2 ?????????????????? Cloud KMS per Tenant ??????????????? KMS key ?????????????????? tenant ?????? AWS/GCP/Azure tenant admin ?????????????????? key ??????????????????????????? platform team ?????????????????????????????? private key ??????????????? tenant ?????? full control ????????????????????? platform team ??????????????????????????? recover ????????? tenant ?????? key age Keys per Tenant platform team generate age key pair ????????? tenant ???????????? private key ????????? public key ??????????????? .sops.yaml ??????????????? ???????????????????????? KMS ????????????????????? key management ????????????????????? tenant ???????????????????????? secrets ????????? ???????????????????????? Cloud KMS ?????????????????? production (managed, audit trail, key rotation built-in)
Q: ???????????????????????????????????? tenant ????????????????????? secrets ????????? tenant ?????????????
A: Isolation ?????????????????????????????????????????? Encryption Level ??????????????? tenant ?????? key ????????? decrypt ???????????????????????? secrets ????????? encrypt ???????????? key ??????????????????????????? ????????????????????????????????? encrypted ???????????????????????????????????? Repository Level ?????????????????????????????? isolation ????????????????????? ????????? Git repo per tenant ?????????????????????????????????????????? tenant ????????????????????? CI/CD Level ????????? OIDC authentication ??????????????? tenant ?????? service account ????????? access ???????????????????????? KMS key ??????????????????????????? Kubernetes Level ??????????????? tenant ???????????????????????? namespace RBAC ????????????????????? cross-namespace access
Q: Key rotation ????????????????????????????????????????????? downtime?
A: SOPS key rotation ??????????????? downtime ??????????????? ????????????????????? 1) Generate new key 2) Update .sops.yaml ??????????????? new key (???????????? old key ?????????????????????) 3) Run sops updatekeys ????????????????????? ??????????????????????????? re-encrypt data key ???????????? new key set ????????? re-encrypt content 4) ??????????????? decrypt ???????????? new key 5) Commit changes 6) CI/CD ????????? new key deploy ???????????? 7) ???????????? verify ??????????????????????????????????????? ?????? old key ????????? .sops.yaml 8) Run sops updatekeys ?????????????????? ????????????????????? rotation ???????????? old key ????????? new key ?????????????????? ??????????????? downtime
