mTLS ?????? Service Mesh ?????????????????????
mTLS (Mutual TLS) ?????? Service Mesh ????????????????????????????????????????????????????????? service-to-service communication ??????????????? cluster ?????????????????????????????????????????????????????????????????????????????? 2 ??????????????????????????????????????? ????????? developer ???????????????????????????????????? code ??????????????? Service Mesh sidecar proxy (Envoy ?????? Istio, linkerd2-proxy ?????? Linkerd) ?????????????????? TLS handshake, certificate provisioning, rotation ??????????????????????????????
Certificate chain ?????? Service Mesh Root CA ???????????? trust anchor ?????????????????????????????? mesh, Intermediate CA (optional) ?????????????????? multi-cluster, Workload certificates ????????????????????????????????? pod/service ????????? SPIFFE identity (spiffe://cluster.local/ns/namespace/sa/service-account) ???????????? identity format ?????????????????????
??????????????????????????? Zero trust network ????????? trust network layer, Encryption in transit ??????????????????????????????????????????????????????????????? services, Identity-based access control ????????? service identity ????????? IP-based, Compliance HIPAA, PCI DSS, SOC 2 ???????????? encryption in transit
????????????????????? Certificate Infrastructure
??????????????? PKI infrastructure ?????????????????? service mesh
# === Certificate Infrastructure Setup ===
# 1. Create Root CA
cat > create_ca.sh << 'BASH'
#!/bin/bash
# Root CA (keep offline, HSM in production)
mkdir -p certs/{root,intermediate,workload}
# Generate Root CA key
openssl genrsa -aes256 -out certs/root/ca-key.pem 4096
# Generate Root CA certificate (10 years)
openssl req -new -x509 -days 3650 -sha384 \
-key certs/root/ca-key.pem \
-out certs/root/ca-cert.pem \
-subj "/CN=Mesh Root CA/O=MyCompany/C=TH" \
-extensions v3_ca \
-config <(cat /etc/ssl/openssl.cnf <(printf "\n[v3_ca]\nbasicConstraints=critical,CA:TRUE\nkeyUsage=critical,keyCertSign,cRLSign"))
# Intermediate CA (for Istio)
openssl genrsa -out certs/intermediate/ca-key.pem 4096
openssl req -new -key certs/intermediate/ca-key.pem \
-out certs/intermediate/ca.csr \
-subj "/CN=Istio CA/O=MyCompany/C=TH"
openssl x509 -req -days 1825 -sha384 \
-in certs/intermediate/ca.csr \
-CA certs/root/ca-cert.pem \
-CAkey certs/root/ca-key.pem \
-CAcreateserial \
-out certs/intermediate/ca-cert.pem \
-extensions v3_ca \
-extfile <(printf "basicConstraints=critical,CA:TRUE,pathlen:0\nkeyUsage=critical,keyCertSign,cRLSign")
# Create certificate chain
cat certs/intermediate/ca-cert.pem certs/root/ca-cert.pem > certs/intermediate/cert-chain.pem
# Verify chain
openssl verify -CAfile certs/root/ca-cert.pem certs/intermediate/ca-cert.pem
echo "Certificate infrastructure created"
BASH
# 2. Install CA into Istio
cat > install_ca.sh << 'BASH'
#!/bin/bash
# Create Kubernetes secret for Istio CA
kubectl create namespace istio-system
kubectl create secret generic cacerts -n istio-system \
--from-file=ca-cert.pem=certs/intermediate/ca-cert.pem \
--from-file=ca-key.pem=certs/intermediate/ca-key.pem \
--from-file=root-cert.pem=certs/root/ca-cert.pem \
--from-file=cert-chain.pem=certs/intermediate/cert-chain.pem
# Install Istio with custom CA
istioctl install --set profile=default \
--set values.global.pilotCertProvider=istiod
echo "Istio CA installed"
BASH
chmod +x create_ca.sh install_ca.sh
echo "PKI infrastructure ready"Istio Certificate Management
?????????????????? certificates ?????? Istio mesh
#!/usr/bin/env python3
# istio_certs.py ??? Istio Certificate Management
import json
import logging
from typing import Dict, List
from datetime import datetime, timedelta
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger("istio_certs")
class IstioCertManager:
"""Manage Istio mesh certificates"""
def __init__(self):
self.workload_certs = {}
def istio_cert_flow(self):
"""How Istio issues workload certificates"""
return {
"step_1": {
"action": "Pod starts with Istio sidecar",
"detail": "istio-proxy (Envoy) container injected by Istio",
},
"step_2": {
"action": "Sidecar generates CSR",
"detail": "Creates private key and Certificate Signing Request",
},
"step_3": {
"action": "CSR sent to istiod (Citadel)",
"detail": "Via SDS (Secret Discovery Service) API",
},
"step_4": {
"action": "istiod validates identity",
"detail": "Checks Kubernetes ServiceAccount token (JWT)",
},
"step_5": {
"action": "istiod signs certificate",
"detail": "Signs with Istio CA, includes SPIFFE ID as SAN",
},
"step_6": {
"action": "Certificate returned to sidecar",
"detail": "Sidecar uses cert for mTLS connections",
},
"step_7": {
"action": "Auto-rotation before expiry",
"detail": "Default: certificates valid 24h, rotate at 50% lifetime",
},
}
def cert_config(self):
"""Istio certificate configuration options"""
return {
"default_workload_cert_ttl": "24h",
"max_workload_cert_ttl": "720h (30 days)",
"root_cert_ttl": "10 years",
"rotation_trigger": "50% of certificate lifetime",
"key_size": "RSA 2048 (default) or ECDSA P-256",
"signature_algorithm": "SHA256",
"spiffe_format": "spiffe://cluster.local/ns/{namespace}/sa/{service-account}",
"mesh_config": {
"defaultConfig": {
"proxyMetadata": {
"SECRET_TTL": "24h",
"SECRET_GRACE_PERIOD_RATIO": "0.5",
},
},
},
}
def multi_cluster_setup(self):
"""Certificate setup for multi-cluster mesh"""
return {
"shared_root_ca": {
"description": "????????? clusters ????????? root CA ????????????????????????",
"setup": "??????????????? root CA ????????????, intermediate CA ?????????????????? cluster",
"benefit": "Cross-cluster mTLS ?????????????????????????????????",
},
"per_cluster_intermediate": {
"cluster_a": "Root CA ??? Intermediate CA-A ??? Workload certs",
"cluster_b": "Root CA ??? Intermediate CA-B ??? Workload certs",
"trust": "???????????? 2 clusters trust ????????????????????? shared root CA",
},
"external_ca": {
"options": ["HashiCorp Vault", "AWS Private CA", "Google CAS"],
"benefit": "Centralized CA management, HSM support",
},
}
manager = IstioCertManager()
flow = manager.istio_cert_flow()
print("Istio Certificate Issuance Flow:")
for step, info in flow.items():
print(f" {step}: {info['action']}")
config = manager.cert_config()
print(f"\nCert Config:")
print(f" Workload TTL: {config['default_workload_cert_ttl']}")
print(f" Rotation: at {config['rotation_trigger']}")
print(f" SPIFFE: {config['spiffe_format']}")cert-manager ????????? Service Mesh
????????? cert-manager ?????????????????? certificates
# === cert-manager + Service Mesh ===
# 1. Install cert-manager
kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.14.0/cert-manager.yaml
# 2. cert-manager as Istio CA (istio-csr)
cat > istio-csr.yaml << 'EOF'
# Install cert-manager istio-csr
# This replaces istiod as certificate issuer
apiVersion: cert-manager.io/v1
kind: Issuer
metadata:
name: istio-ca
namespace: istio-system
spec:
ca:
secretName: istio-ca-secret
---
# Or use Let's Encrypt for external certificates
apiVersion: cert-manager.io/v1
kind: ClusterIssuer
metadata:
name: letsencrypt-prod
spec:
acme:
server: https://acme-v02.api.letsencrypt.org/directory
email: admin@example.com
privateKeySecretRef:
name: letsencrypt-prod-key
solvers:
- http01:
ingress:
class: nginx
---
# Certificate for Ingress Gateway
apiVersion: cert-manager.io/v1
kind: Certificate
metadata:
name: gateway-cert
namespace: istio-system
spec:
secretName: gateway-tls
issuerRef:
name: letsencrypt-prod
kind: ClusterIssuer
dnsNames:
- api.example.com
- "*.api.example.com"
renewBefore: 720h
---
# Internal CA Certificate
apiVersion: cert-manager.io/v1
kind: Certificate
metadata:
name: internal-service-cert
namespace: production
spec:
secretName: service-tls
issuerRef:
name: istio-ca
kind: Issuer
commonName: myservice.production.svc.cluster.local
dnsNames:
- myservice
- myservice.production
- myservice.production.svc.cluster.local
duration: 24h
renewBefore: 12h
privateKey:
algorithm: ECDSA
size: 256
EOF
kubectl apply -f istio-csr.yaml
# 3. Verify certificates
kubectl get certificates -A
kubectl describe certificate gateway-cert -n istio-system
# 4. Check certificate details
kubectl get secret gateway-tls -n istio-system -o jsonpath='{.data.tls\.crt}' | \
base64 -d | openssl x509 -noout -text
echo "cert-manager configured"HashiCorp Vault PKI Integration
????????? Vault ???????????? CA ?????????????????? service mesh
# === HashiCorp Vault PKI for Service Mesh ===
# 1. Enable Vault PKI
cat > vault_pki_setup.sh << 'BASH'
#!/bin/bash
# Enable PKI secrets engine
vault secrets enable pki
vault secrets tune -max-lease-ttl=87600h pki
# Generate Root CA
vault write pki/root/generate/internal \
common_name="Mesh Root CA" \
ttl=87600h \
key_type=rsa \
key_bits=4096
# Configure URLs
vault write pki/config/urls \
issuing_certificates="https://vault.example.com:8200/v1/pki/ca" \
crl_distribution_points="https://vault.example.com:8200/v1/pki/crl"
# Enable Intermediate CA for Istio
vault secrets enable -path=pki_istio pki
vault secrets tune -max-lease-ttl=43800h pki_istio
# Generate Intermediate CSR
vault write pki_istio/intermediate/generate/internal \
common_name="Istio Intermediate CA" \
ttl=43800h
# Sign with Root CA
vault write pki/root/sign-intermediate \
csr=@intermediate.csr \
format=pem_bundle \
ttl=43800h
# Create role for workload certificates
vault write pki_istio/roles/istio-workload \
allowed_domains="svc.cluster.local" \
allow_subdomains=true \
max_ttl=72h \
ttl=24h \
key_type=ec \
key_bits=256 \
require_cn=false \
allowed_uri_sans="spiffe://cluster.local/*"
echo "Vault PKI configured"
BASH
# 2. Vault Agent Injector for cert delivery
cat > vault_agent.yaml << 'EOF'
apiVersion: apps/v1
kind: Deployment
metadata:
name: myservice
spec:
template:
metadata:
annotations:
vault.hashicorp.com/agent-inject: "true"
vault.hashicorp.com/role: "istio-workload"
vault.hashicorp.com/agent-inject-secret-cert: "pki_istio/issue/istio-workload"
vault.hashicorp.com/agent-inject-template-cert: |
{{- with secret "pki_istio/issue/istio-workload"
"common_name=myservice.production.svc.cluster.local"
"ttl=24h" -}}
{{ .Data.certificate }}
{{ .Data.issuing_ca }}
{{- end }}
EOF
echo "Vault integration configured"Monitoring ????????? Rotation Automation
?????????????????????????????????????????????????????? certificates ???????????????????????????
#!/usr/bin/env python3
# cert_monitoring.py ??? Certificate Monitoring for Service Mesh
import json
import logging
from typing import Dict, List
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger("monitor")
class CertMonitoring:
def __init__(self):
pass
def prometheus_queries(self):
return {
"cert_expiry": {
"query": 'citadel_server_root_cert_expiry_timestamp - time()',
"description": "Seconds until root cert expires",
"alert_threshold": 2592000,
},
"workload_cert_rotation": {
"query": 'rate(citadel_server_csr_count[5m])',
"description": "Certificate signing rate",
},
"csr_errors": {
"query": 'rate(citadel_server_csr_parsing_err_count[5m])',
"description": "CSR parsing errors",
"alert_threshold": 0,
},
"mtls_connections": {
"query": 'istio_tcp_connections_opened_total{connection_security_policy="mutual_tls"}',
"description": "mTLS connections count",
},
"plaintext_connections": {
"query": 'istio_tcp_connections_opened_total{connection_security_policy="none"}',
"description": "Non-mTLS connections (should be 0 in STRICT mode)",
"alert_threshold": 0,
},
}
def alert_rules(self):
return [
{
"name": "RootCertExpiringSoon",
"expr": "citadel_server_root_cert_expiry_timestamp - time() < 2592000",
"severity": "critical",
"message": "Root certificate expires in less than 30 days",
},
{
"name": "WorkloadCertRotationFailing",
"expr": "rate(citadel_server_csr_parsing_err_count[5m]) > 0",
"severity": "warning",
"message": "Workload certificate rotation errors detected",
},
{
"name": "PlaintextTrafficDetected",
"expr": 'rate(istio_tcp_connections_opened_total{connection_security_policy="none"}[5m]) > 0',
"severity": "critical",
"message": "Non-mTLS traffic detected in STRICT mode namespace",
},
]
def rotation_best_practices(self):
return {
"workload_certs": {
"ttl": "24 hours (default, recommended)",
"rotation": "Automatic at 50% lifetime (12h)",
"action": "No manual intervention needed",
},
"intermediate_ca": {
"ttl": "5 years",
"rotation": "Plan rotation 6 months before expiry",
"action": "Generate new intermediate, update Kubernetes secret, rolling restart",
},
"root_ca": {
"ttl": "10 years",
"rotation": "Plan rotation 1 year before expiry",
"action": "Most complex: trust both old and new root during transition period",
},
}
monitor = CertMonitoring()
queries = monitor.prometheus_queries()
print("Prometheus Monitoring Queries:")
for name, info in queries.items():
print(f" {name}: {info['description']}")
practices = monitor.rotation_best_practices()
print("\nRotation Best Practices:")
for cert_type, info in practices.items():
print(f" {cert_type}: TTL={info['ttl']}, Rotation={info['rotation']}")FAQ ??????????????????????????????????????????
Q: Istio CA ????????? External CA (Vault/cert-manager) ??????????????????????????????????
A: Istio built-in CA (Citadel/istiod) ????????????????????????????????? ??????????????????????????????????????????, single cluster, ?????????????????????????????? centralized CA management ??????????????? ???????????? ???????????????????????????????????????????????? setup ??????????????? ????????????????????? key ?????????????????? Kubernetes secret (???????????????????????? production ??????????????????????????????) External CA (Vault, AWS Private CA, cert-manager) ????????????????????????????????? multi-cluster mesh, compliance ????????????????????? HSM, centralized certificate management, integration ????????? existing PKI ??????????????? key ?????????????????? HSM audit logging ????????? ????????????????????? ????????????????????????????????? ???????????? maintain external system ??????????????? ??????????????????????????? Istio CA ??????????????? ready ?????????????????????????????? Vault ???????????? cert-manager istio-csr
Q: SPIFFE ????????????????????? ????????????????????????????
A: SPIFFE (Secure Production Identity Framework For Everyone) ???????????? standard ?????????????????? workload identity ????????? URI format spiffe://trust-domain/path ???????????? spiffe://cluster.local/ns/production/sa/api-server ?????????????????????????????? Platform-agnostic ?????????????????????????????? Kubernetes, VMs, bare metal, Identity-based ??????????????????????????? IP (???????????????????????????????????????????????? cloud), Interoperable Istio, Linkerd, SPIRE ????????? SPIFFE ??????????????????????????????, Cross-cluster identity ????????? SPIFFE ???????????? clusters ????????? ?????? Istio SPIFFE ID ????????? encode ???????????? SAN (Subject Alternative Name) ?????? X.509 certificate Authorization policies ????????? SPIFFE ID ???????????????????????? service ?????????????????????????????? service ??????????????????
Q: Certificate rotation ??????????????? connection drop ??????????
A: ????????? Istio ??????????????????????????? rotation ???????????????????????? traffic Certificate ????????????????????? issue ???????????? cert ????????????????????????????????? (rotate ????????? 50% lifetime) Envoy sidecar ?????????????????? hot certificate reload ????????????????????? restart proxy Existing connections ?????????????????? cert ??????????????????????????????????????? connection ??????????????? Connections ????????????????????? cert ??????????????????????????????????????? ??????????????????????????????????????????????????? istiod ????????? ??????????????????????????? sign cert ????????????????????? (?????????: HA istiod), Clock skew ????????????????????? nodes ??????????????? cert ???????????????????????? expired (?????????: NTP sync), Resource limits istiod ??????????????? resources ?????? sign cert ???????????????????????? (?????????: ??????????????? resources)
Q: ??????????????? mTLS ??????????????????????????????????????????????????????????
A: ???????????????????????? istioctl authn tls-check ????????????????????? mTLS status ?????????????????? services, kubectl exec ???????????? pod ????????? curl ??????????????? service ????????????????????????/?????????????????? cert, istioctl proxy-config secret ?????? certificates ????????? sidecar ?????????????????????, Kiali dashboard ???????????? mTLS icon ??????????????? connections, ??????????????? negative case ????????? call ????????? pod ???????????????????????? sidecar ??????????????? STRICT namespace ???????????? fail, Prometheus query ????????????????????????????????????????????? plaintext connections ?????????????????? automated testing ????????? istioctl analyze ???????????? configuration errors ????????? policy violations ?????? CI/CD
