Ceph Storage Cluster คืออะไร
Ceph เป็น open source distributed storage system ที่ให้บริการ object storage, block storage และ file storage บน single unified platform ออกแบบมาเพื่อ high availability, scalability และ self-healing ใช้ CRUSH algorithm สำหรับ data placement ไม่มี single point of failure
Components หลักของ Ceph ได้แก่ MON (Monitor) จัดการ cluster map และ consensus ต้องมีอย่างน้อย 3 nodes, OSD (Object Storage Daemon) เก็บข้อมูลจริง แต่ละ OSD ดูแล disk หนึ่งลูก, MDS (Metadata Server) จัดการ metadata สำหรับ CephFS, MGR (Manager) จัดการ monitoring, dashboard และ orchestration, RGW (RADOS Gateway) ให้บริการ S3/Swift compatible object storage
Capacity Planning สำหรับ Ceph สำคัญมากเพราะ replication factor กระทบ usable capacity โดยตรง (3x replication ใช้ raw space 3 เท่า), OSD placement groups ต้อง size ถูกต้องเพื่อ performance ที่ดี, network bandwidth ต้องเพียงพอสำหรับ replication traffic, recovery จาก disk failure ต้องมี spare capacity และ growth projection ต้องวางแผนล่วงหน้า 6-12 เดือน
ติดตั้ง Ceph Cluster
วิธีติดตั้ง Ceph ด้วย cephadm
# === ติดตั้ง Ceph Cluster ด้วย cephadm ===
# Prerequisites:
# - 3+ nodes with Ubuntu 22.04 or Rocky Linux 9
# - Each node: 4+ CPU cores, 8+ GB RAM, 1+ SSD/NVMe for OSD
# - Network: 10GbE recommended (minimum 1GbE)
# - Separate public and cluster networks recommended
# 1. Install cephadm on first node
curl --silent --remote-name --location https://download.ceph.com/rpm-reef/el9/noarch/cephadm
chmod +x cephadm
./cephadm add-repo --release reef
./cephadm install
# 2. Bootstrap cluster
cephadm bootstrap \
--mon-ip 10.0.1.10 \
--cluster-network 10.0.2.0/24 \
--dashboard-password-noupdate \
--initial-dashboard-password MyDashP@ss
# Dashboard: https://10.0.1.10:8443
# Default user: admin
# 3. Add hosts
# Copy SSH key to other nodes first
ssh-copy-id root@ceph-node2
ssh-copy-id root@ceph-node3
ceph orch host add ceph-node2 10.0.1.11
ceph orch host add ceph-node3 10.0.1.12
# 4. Add OSDs (auto-detect available disks)
ceph orch apply osd --all-available-devices
# Or specific devices
ceph orch daemon add osd ceph-node1:/dev/sdb
ceph orch daemon add osd ceph-node2:/dev/sdb
ceph orch daemon add osd ceph-node3:/dev/sdb
# 5. Set MON count
ceph orch apply mon --placement="3 ceph-node1 ceph-node2 ceph-node3"
# 6. Enable features
ceph mgr module enable dashboard
ceph mgr module enable prometheus
ceph mgr module enable pg_autoscaler
# 7. Create pools
ceph osd pool create rbd-pool 128
ceph osd pool set rbd-pool size 3
ceph osd pool set rbd-pool min_size 2
ceph osd pool application enable rbd-pool rbd
# Create CephFS
ceph fs volume create cephfs
# Create RGW (S3)
ceph orch apply rgw myrgw --placement="2 ceph-node1 ceph-node2" --port=7480
# 8. Verify cluster
ceph -s
# cluster:
# id: abc123
# health: HEALTH_OK
# services:
# mon: 3 daemons
# mgr: 2 active
# osd: 9 osds: 9 up, 9 in
# rgw: 2 daemons active
ceph osd tree
ceph df
echo "Ceph cluster installed"Capacity Planning และ Sizing
เครื่องมือคำนวณ capacity
#!/usr/bin/env python3
# ceph_capacity_planner.py — Ceph Cluster Capacity Planning
import json
import math
import logging
from typing import Dict, List
from dataclasses import dataclass
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger("capacity")
@dataclass
class DiskSpec:
size_tb: float
type: str # hdd, ssd, nvme
rpm: int = 0 # HDD only
iops: int = 0
throughput_mbps: int = 0
@dataclass
class NodeSpec:
hostname: str
cpu_cores: int
ram_gb: int
disks: List[DiskSpec]
network_gbps: float = 10
class CephCapacityPlanner:
def __init__(self):
self.nodes: List[NodeSpec] = []
def add_node(self, node: NodeSpec):
self.nodes.append(node)
def calculate_raw_capacity(self):
total_tb = 0
by_type = {"hdd": 0, "ssd": 0, "nvme": 0}
osd_count = 0
for node in self.nodes:
for disk in node.disks:
total_tb += disk.size_tb
by_type[disk.type] += disk.size_tb
osd_count += 1
return {
"total_raw_tb": round(total_tb, 1),
"by_type": {k: round(v, 1) for k, v in by_type.items() if v > 0},
"osd_count": osd_count,
"node_count": len(self.nodes),
}
def calculate_usable_capacity(self, replication_factor=3,
overhead_pct=5, max_fill_pct=85):
raw = self.calculate_raw_capacity()
raw_tb = raw["total_raw_tb"]
# After replication
after_replication = raw_tb / replication_factor
# Reserved for overhead (BlueStore, metadata)
after_overhead = after_replication * (1 - overhead_pct / 100)
# Max usable (never fill above 85%)
usable = after_overhead * (max_fill_pct / 100)
return {
"raw_tb": round(raw_tb, 1),
"replication_factor": replication_factor,
"after_replication_tb": round(after_replication, 1),
"overhead_pct": overhead_pct,
"after_overhead_tb": round(after_overhead, 1),
"max_fill_pct": max_fill_pct,
"usable_tb": round(usable, 1),
"efficiency_pct": round(usable / raw_tb * 100, 1),
"osd_count": raw["osd_count"],
}
def calculate_pg_count(self, pool_pct_of_data=100, osd_count=None,
replication_factor=3, target_pgs_per_osd=100):
if osd_count is None:
osd_count = self.calculate_raw_capacity()["osd_count"]
# Formula: (target_pgs_per_osd * osd_count * pool_pct / 100) / replication_factor
ideal = (target_pgs_per_osd * osd_count * pool_pct_of_data / 100) / replication_factor
# Round to nearest power of 2
pg_count = 2 ** math.ceil(math.log2(max(ideal, 1)))
return {
"osd_count": osd_count,
"pool_data_pct": pool_pct_of_data,
"replication_factor": replication_factor,
"ideal_pgs": round(ideal),
"recommended_pg_count": pg_count,
}
def forecast_growth(self, current_usage_tb, monthly_growth_tb,
months=12):
capacity = self.calculate_usable_capacity()
usable = capacity["usable_tb"]
forecast = []
for month in range(1, months + 1):
projected = current_usage_tb + (monthly_growth_tb * month)
utilization = projected / usable * 100
if utilization >= 85:
status = "CRITICAL"
elif utilization >= 70:
status = "WARNING"
else:
status = "OK"
forecast.append({
"month": month,
"projected_tb": round(projected, 1),
"utilization_pct": round(utilization, 1),
"free_tb": round(usable - projected, 1),
"status": status,
})
# Find when capacity runs out
months_until_full = None
for f in forecast:
if f["utilization_pct"] >= 85:
months_until_full = f["month"]
break
return {
"current_usage_tb": current_usage_tb,
"usable_capacity_tb": usable,
"monthly_growth_tb": monthly_growth_tb,
"months_until_85pct": months_until_full,
"forecast": forecast,
}
def recommend_hardware(self, target_usable_tb, workload="mixed"):
configs = {
"archive": {
"disk": DiskSpec(18, "hdd", rpm=7200, iops=100, throughput_mbps=200),
"disks_per_node": 12, "nodes_min": 3, "replication": 3,
},
"mixed": {
"disk": DiskSpec(4, "ssd", iops=50000, throughput_mbps=500),
"disks_per_node": 6, "nodes_min": 3, "replication": 3,
},
"performance": {
"disk": DiskSpec(2, "nvme", iops=200000, throughput_mbps=3000),
"disks_per_node": 4, "nodes_min": 3, "replication": 3,
},
}
config = configs.get(workload, configs["mixed"])
raw_needed = target_usable_tb / (1 / config["replication"]) / 0.80
disks_needed = math.ceil(raw_needed / config["disk"].size_tb)
nodes_needed = max(
math.ceil(disks_needed / config["disks_per_node"]),
config["nodes_min"],
)
actual_disks = nodes_needed * config["disks_per_node"]
actual_raw = actual_disks * config["disk"].size_tb
actual_usable = actual_raw / config["replication"] * 0.80
return {
"workload": workload,
"target_usable_tb": target_usable_tb,
"recommendation": {
"nodes": nodes_needed,
"disks_per_node": config["disks_per_node"],
"total_disks": actual_disks,
"disk_type": config["disk"].type,
"disk_size_tb": config["disk"].size_tb,
"raw_capacity_tb": round(actual_raw, 1),
"usable_capacity_tb": round(actual_usable, 1),
"replication": config["replication"],
},
}
planner = CephCapacityPlanner()
for i in range(3):
planner.add_node(NodeSpec(
hostname=f"ceph-node{i+1}",
cpu_cores=16, ram_gb=64,
disks=[DiskSpec(4, "ssd", iops=50000) for _ in range(6)],
network_gbps=25,
))
print("Raw:", json.dumps(planner.calculate_raw_capacity(), indent=2))
print("Usable:", json.dumps(planner.calculate_usable_capacity(), indent=2))
print("PGs:", json.dumps(planner.calculate_pg_count(pool_pct_of_data=80), indent=2))
print("Forecast:", json.dumps(planner.forecast_growth(10, 2, months=12), indent=2))
print("Recommend:", json.dumps(planner.recommend_hardware(50, "mixed"), indent=2))Monitoring Cluster Health
Monitor Ceph cluster health และ performance
#!/usr/bin/env python3
# ceph_monitor.py — Ceph Cluster Health Monitoring
import subprocess
import json
import logging
from datetime import datetime
from typing import Dict, List
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger("ceph_mon")
class CephMonitor:
def __init__(self):
self.history = []
def _run_ceph(self, *args):
cmd = ["ceph", "--format", "json"] + list(args)
result = subprocess.run(cmd, capture_output=True, text=True)
if result.returncode != 0:
logger.error(f"ceph command failed: {result.stderr}")
return {}
return json.loads(result.stdout)
def get_cluster_status(self):
status = self._run_ceph("status")
health = status.get("health", {})
osd_map = status.get("osdmap", {}).get("osdmap", {})
pg_map = status.get("pgmap", {})
return {
"timestamp": datetime.utcnow().isoformat(),
"health": health.get("status", "UNKNOWN"),
"health_checks": list(health.get("checks", {}).keys()),
"osds": {
"total": osd_map.get("num_osds", 0),
"up": osd_map.get("num_up_osds", 0),
"in": osd_map.get("num_in_osds", 0),
},
"pgs": {
"total": pg_map.get("num_pgs", 0),
"active_clean": pg_map.get("num_pgs_by_state", [{}])[0].get("count", 0)
if pg_map.get("num_pgs_by_state") else 0,
},
"storage": {
"total_bytes": pg_map.get("bytes_total", 0),
"used_bytes": pg_map.get("bytes_used", 0),
"available_bytes": pg_map.get("bytes_avail", 0),
"used_pct": round(
pg_map.get("bytes_used", 0) / max(pg_map.get("bytes_total", 1), 1) * 100, 1
),
},
"io": {
"read_bytes_sec": pg_map.get("read_bytes_sec", 0),
"write_bytes_sec": pg_map.get("write_bytes_sec", 0),
"read_op_sec": pg_map.get("read_op_per_sec", 0),
"write_op_sec": pg_map.get("write_op_per_sec", 0),
},
}
def get_osd_status(self):
osds = self._run_ceph("osd", "df")
osd_list = []
for osd in osds.get("nodes", []):
utilization = osd.get("utilization", 0)
status = "ok"
if utilization > 85:
status = "critical"
elif utilization > 75:
status = "warning"
osd_list.append({
"id": osd.get("id"),
"name": osd.get("name"),
"total_kb": osd.get("kb"),
"used_kb": osd.get("kb_used"),
"available_kb": osd.get("kb_avail"),
"utilization_pct": round(utilization, 1),
"status": status,
})
return sorted(osd_list, key=lambda x: x["utilization_pct"], reverse=True)
def get_pool_stats(self):
pools = self._run_ceph("osd", "pool", "stats")
pool_list = []
for pool in pools:
pool_list.append({
"name": pool.get("pool_name"),
"id": pool.get("pool_id"),
"stored_bytes": pool.get("stored", 0),
"objects": pool.get("objects", 0),
"read_bytes_sec": pool.get("rd_bytes", 0),
"write_bytes_sec": pool.get("wr_bytes", 0),
})
return pool_list
def health_check(self):
status = self.get_cluster_status()
osds = self.get_osd_status()
issues = []
if status["health"] != "HEALTH_OK":
issues.append(f"Cluster health: {status['health']}")
for check in status["health_checks"]:
issues.append(f" - {check}")
down_osds = status["osds"]["total"] - status["osds"]["up"]
if down_osds > 0:
issues.append(f"{down_osds} OSDs are down")
if status["storage"]["used_pct"] > 80:
issues.append(f"Storage usage: {status['storage']['used_pct']}%")
for osd in osds:
if osd["status"] == "critical":
issues.append(f"OSD {osd['name']} at {osd['utilization_pct']}%")
return {
"timestamp": status["timestamp"],
"healthy": len(issues) == 0,
"issues": issues,
"summary": status,
}
# monitor = CephMonitor()
# health = monitor.health_check()
# print(json.dumps(health, indent=2))Performance Tuning และ Optimization
ปรับแต่ง Ceph performance
# === Ceph Performance Tuning ===
# 1. BlueStore Tuning
# ===================================
# Dedicated WAL/DB on NVMe for HDD OSDs
ceph osd set-full-ratio 0.90
ceph osd set-nearfull-ratio 0.80
ceph osd set-backfillfull-ratio 0.85
# Per-OSD settings in ceph.conf
# [osd]
# bluestore_cache_size = 4294967296 # 4GB cache per OSD
# bluestore_cache_autotune = true
# bluestore_cache_meta_ratio = 0.4
# bluestore_cache_kv_ratio = 0.4
# bluestore_cache_data_ratio = 0.2
#
# # Async IO
# bdev_async_discard = true
# bdev_enable_discard = true
#
# # Recovery tuning (balance between recovery speed and client IO)
# osd_recovery_max_active = 3
# osd_recovery_max_active_hdd = 1
# osd_recovery_max_active_ssd = 10
# osd_max_backfills = 1
# osd_recovery_sleep = 0
# osd_recovery_priority = 5
# 2. Network Tuning
# ===================================
# Separate public and cluster networks
# [global]
# public_network = 10.0.1.0/24
# cluster_network = 10.0.2.0/24
# ms_type = async+posix
# ms_async_op_threads = 5
# System network tuning
sudo sysctl -w net.core.rmem_max=67108864
sudo sysctl -w net.core.wmem_max=67108864
sudo sysctl -w net.core.rmem_default=33554432
sudo sysctl -w net.core.wmem_default=33554432
sudo sysctl -w net.ipv4.tcp_rmem="4096 87380 33554432"
sudo sysctl -w net.ipv4.tcp_wmem="4096 65536 33554432"
# 3. PG Autoscaler
# ===================================
ceph mgr module enable pg_autoscaler
ceph osd pool set rbd-pool pg_autoscale_mode on
# Check PG status
ceph osd pool autoscale-status
# 4. Pool Optimization
# ===================================
# Set compression for suitable pools
ceph osd pool set archive-pool compression_algorithm snappy
ceph osd pool set archive-pool compression_mode aggressive
ceph osd pool set archive-pool compression_required_ratio 0.875
# 5. Benchmark
# ===================================
# RADOS bench (sequential write)
rados bench -p rbd-pool 60 write --no-cleanup
# RADOS bench (sequential read)
rados bench -p rbd-pool 60 seq
# RADOS bench (random read)
rados bench -p rbd-pool 60 rand
# RBD bench
rbd create test-image --size 10G --pool rbd-pool
rbd bench test-image --io-type write --io-size 4K --io-threads 16 --pool rbd-pool
rbd bench test-image --io-type read --io-size 4K --io-threads 16 --pool rbd-pool
# fio with RBD
# [global]
# ioengine=rbd
# clientname=admin
# pool=rbd-pool
# rbdname=test-image
# rw=randread
# bs=4k
# numjobs=4
# iodepth=32
# runtime=60
echo "Ceph tuning complete"Scaling และ Maintenance
ขยาย cluster และ maintenance
# === Ceph Cluster Scaling ===
# 1. Add New OSD Host
# ===================================
# Add host to cluster
ceph orch host add ceph-node4 10.0.1.13
# Add OSDs on new host
ceph orch daemon add osd ceph-node4:/dev/sdb
ceph orch daemon add osd ceph-node4:/dev/sdc
ceph orch daemon add osd ceph-node4:/dev/sdd
# Monitor rebalancing
ceph -w
# Watch PG states: active+remapped, backfilling, recovering
# Wait until all PGs are active+clean
# 2. Remove OSD (Graceful)
# ===================================
# Mark OSD out (starts rebalancing)
ceph osd out osd.5
# Wait for rebalancing
while ! ceph osd safe-to-destroy osd.5; do
echo "Waiting for safe-to-destroy..."
sleep 30
done
# Stop and remove
ceph orch daemon rm osd.5
ceph osd purge osd.5 --yes-i-really-mean-it
# 3. Replace Failed Disk
# ===================================
# Identify failed OSD
ceph osd tree | grep down
ceph health detail
# Remove failed OSD
ceph osd out osd.3
ceph osd purge osd.3 --yes-i-really-mean-it
# Replace disk physically, then add new OSD
ceph orch daemon add osd ceph-node2:/dev/sdc
# 4. Maintenance Window
# ===================================
# Set noout flag (prevent rebalancing during maintenance)
ceph osd set noout
# Perform maintenance...
# Restart services, update firmware, etc.
sudo systemctl restart ceph-osd@3
# Unset flag after maintenance
ceph osd unset noout
# 5. Upgrade Ceph
# ===================================
# Check current version
ceph versions
# Set container image for upgrade
ceph orch upgrade start --image quay.io/ceph/ceph:v18.2.2
# Monitor upgrade progress
ceph orch upgrade status
# 6. Backup Strategies
# ===================================
# RBD snapshots
rbd snap create rbd-pool/myimage@snap1
rbd snap ls rbd-pool/myimage
# Export snapshot
rbd export rbd-pool/myimage@snap1 /backup/myimage-snap1.img
# Incremental export
rbd export-diff rbd-pool/myimage@snap1 --from-snap snap0 /backup/myimage-diff.img
# CephFS snapshots
mkdir /mnt/cephfs/.snap/daily-$(date +%Y%m%d)
# 7. Monitoring Alerts
# ===================================
# Prometheus alerting rules for Ceph
# groups:
# - name: ceph
# rules:
# - alert: CephHealthWarning
# expr: ceph_health_status == 1
# for: 5m
# - alert: CephHealthError
# expr: ceph_health_status == 2
# for: 1m
# - alert: CephOSDDown
# expr: count(ceph_osd_up == 0) > 0
# for: 5m
# - alert: CephStorageFull
# expr: ceph_cluster_total_used_bytes / ceph_cluster_total_bytes > 0.80
# for: 10m
echo "Ceph scaling and maintenance documented"FAQ คำถามที่พบบ่อย
Q: Ceph ต้องการ hardware อะไรบ้าง?
A: ขั้นต่ำสำหรับ production 3 nodes แต่ละ node ต้องมี CPU 4+ cores (แนะนำ 1 core ต่อ OSD), RAM 4GB base + 5GB ต่อ OSD (เช่น 6 OSDs ต้อง 34GB), 1 SSD/NVMe ต่อ OSD สำหรับ data (หรือ HDD กับ NVMe WAL/DB), network 10GbE ขั้นต่ำ (25GbE สำหรับ NVMe clusters) SSD สำหรับ MON และ MGR (ไม่ต้องใหญ่ 50-100GB พอ) สำหรับ production จริงจัง แยก public network และ cluster network
Q: Replication 3 กับ Erasure Coding ต่างกันอย่างไร?
A: Replication 3 เก็บ data 3 copies ใช้ raw space 3 เท่า แต่ recovery เร็ว read performance ดี (อ่านจาก copy ไหนัก็ได้) เหมาะสำหรับ hot data ที่ต้องการ low latency Erasure Coding (เช่น k=4, m=2) ใช้ raw space 1.5 เท่า ประหยัดกว่ามาก แต่ write latency สูงกว่า, recovery ช้ากว่า, ใช้ CPU มากกว่า เหมาะสำหรับ cold/archive data ที่ต้องการ capacity efficiency
Q: PG count ตั้งเท่าไหรดี?
A: ใช้สูตร (target_pgs_per_osd * total_osds * pool_pct) / replication_factor แล้ว round up เป็น power of 2 เช่น 18 OSDs, replication 3, pool ใช้ 80% ของ data = (100 * 18 * 0.8) / 3 = 480 ปัดเป็น 512 PGs ต่ำกว่า 100 PGs per OSD จะเสีย performance สูงกว่า 200 PGs per OSD จะใช้ memory มาก Ceph Reef+ มี pg_autoscaler ที่ adjust อัตโนมัติ แนะนำเปิดใช้
Q: Ceph เหมาะกับ workload ประเภทไหน?
A: เหมาะมากสำหรับ object storage (S3 compatible) ใช้แทน MinIO สำหรับ large scale, block storage สำหรับ VMs (Proxmox, OpenStack), Kubernetes persistent volumes (Rook-Ceph), backup storage และ data lake ไม่เหมาะสำหรับ workloads ที่ต้องการ ultra-low latency (ใช้ local NVMe แทน), small clusters น้อยกว่า 3 nodes, single-tenant database ที่ต้องการ consistent latency