Parquet Format Capacity Planning

Parquet Capacity Planning

Parquet Format Capacity Planning Columnar Storage Compression Row Group Partition Strategy Data Lake Storage Estimation Production

Storage Estimation

# === Parquet Storage Calculator ===

from dataclasses import dataclass

@dataclass
class Column:
 name: str
 dtype: str
 avg_bytes_raw: float
 compression_ratio: float

@dataclass
class StorageEstimate:
 description: str
 rows_per_day: int
 columns: list
 retention_days: int
 replicas: int

def estimate_storage(est):
 raw_per_row = sum(c.avg_bytes_raw for c in est.columns)
 compressed_per_row = sum(c.avg_bytes_raw * (1 - c.compression_ratio) for c in est.columns)

 raw_daily_gb = (raw_per_row * est.rows_per_day) / (1024**3)
 compressed_daily_gb = (compressed_per_row * est.rows_per_day) / (1024**3)
 total_gb = compressed_daily_gb * est.retention_days * est.replicas

 print(f" [{est.description}]")
 print(f" Rows/day: {est.rows_per_day:,} | Columns: {len(est.columns)}")
 print(f" Raw/row: {raw_per_row:.0f} bytes | Compressed/row: {compressed_per_row:.1f} bytes")
 print(f" Raw/day: {raw_daily_gb:.1f} GB | Compressed/day: {compressed_daily_gb:.2f} GB")
 print(f" Retention: {est.retention_days} days × {est.replicas} replicas")
 print(f" Total Storage: {total_gb:.1f} GB ({total_gb/1024:.2f} TB)")
 return total_gb

# Example: E-commerce Event Log
ecommerce_cols = [
 Column("event_id", "STRING", 36, 0.70),
 Column("timestamp", "TIMESTAMP", 8, 0.85),
 Column("user_id", "INT64", 8, 0.80),
 Column("event_type", "ENUM", 15, 0.95),
 Column("product_id", "INT64", 8, 0.75),
 Column("category", "ENUM", 20, 0.95),
 Column("price", "DOUBLE", 8, 0.40),
 Column("quantity", "INT32", 4, 0.80),
 Column("session_id", "STRING", 36, 0.65),
 Column("device", "ENUM", 10, 0.95),
 Column("country", "ENUM", 5, 0.97),
 Column("page_url", "STRING", 100, 0.75),
]

print("=== Storage Estimates ===\n")
e1 = StorageEstimate("E-commerce Events", 50_000_000, ecommerce_cols, 365, 2)
total = estimate_storage(e1)

# Cost estimation
s3_cost_per_gb = 0.023
monthly_cost = (total / 12) * s3_cost_per_gb
print(f"\n S3 Cost: /month (/GB)")

Row Group and Compression

# === Row Group Optimization ===

# PyArrow — Write optimized Parquet
# import pyarrow as pa
# import pyarrow.parquet as pq
#
# table = pa.Table.from_pandas(df)
# pq.write_table(table, 'data.parquet',
# compression='zstd',
# compression_level=3,
# row_group_size=1_000_000, # rows per group
# use_dictionary=True, # dictionary encoding
# write_statistics=True, # min/max stats for pushdown
# )

# Spark — Parquet settings
# spark.conf.set("spark.sql.parquet.compression.codec", "zstd")
# spark.conf.set("spark.sql.parquet.block.size", str(256 * 1024 * 1024))
# spark.conf.set("spark.sql.parquet.enableVectorizedReader", "true")
# spark.conf.set("spark.sql.parquet.filterPushdown", "true")

# DuckDB — Read Parquet efficiently
# SELECT product_id, SUM(price * quantity) as revenue
# FROM read_parquet('s3://data/events/year=2024/**/*.parquet',
# hive_partitioning=true)
# WHERE event_type = 'purchase'
# AND year = 2024 AND month = 6
# GROUP BY product_id
# ORDER BY revenue DESC
# LIMIT 100;

@dataclass
class CompressionBenchmark:
 codec: str
 ratio: str
 write_speed: str
 read_speed: str
 cpu_usage: str
 best_for: str

benchmarks = [
 CompressionBenchmark("Snappy", "60-70%", "Fast (500 MB/s)", "Fast (800 MB/s)", "Low",
 "Default, balanced speed/compression"),
 CompressionBenchmark("ZSTD (level 3)", "75-85%", "Medium (300 MB/s)", "Fast (700 MB/s)", "Medium",
 "Best overall, good compression + speed"),
 CompressionBenchmark("ZSTD (level 9)", "80-90%", "Slow (100 MB/s)", "Fast (700 MB/s)", "High",
 "Cold storage, archive"),
 CompressionBenchmark("Gzip", "75-85%", "Slow (80 MB/s)", "Medium (400 MB/s)", "High",
 "Compatibility, legacy systems"),
 CompressionBenchmark("LZ4", "55-65%", "Very fast (700 MB/s)", "Very fast (1 GB/s)", "Very low",
 "Hot data, latency-sensitive"),
 CompressionBenchmark("Uncompressed", "0%", "Very fast", "Very fast", "None",
 "Temporary data, debugging"),
]

print("\n=== Compression Benchmarks ===")
for b in benchmarks:
 print(f" [{b.codec}] Ratio: {b.ratio}")
 print(f" Write: {b.write_speed} | Read: {b.read_speed}")
 print(f" CPU: {b.cpu_usage} | Best for: {b.best_for}")

Partition and Lifecycle

# === Partition Strategy ===

@dataclass
class PartitionPlan:
 table: str
 partition_by: str
 files_per_partition: str
 file_size: str
 compaction: str

plans = [
 PartitionPlan("events", "year/month/day", "1-10 files", "128 MB - 1 GB",
 "Daily compaction at 02:00 UTC"),
 PartitionPlan("users", "country", "1-5 files", "256 MB - 1 GB",
 "Weekly compaction"),
 PartitionPlan("transactions", "year/month", "5-20 files", "256 MB - 1 GB",
 "Monthly compaction"),
 PartitionPlan("logs", "year/month/day/hour", "1-3 files", "64 MB - 512 MB",
 "Hourly micro-batch, daily compaction"),
]

print("=== Partition Plans ===")
for p in plans:
 print(f" [{p.table}] Partition: {p.partition_by}")
 print(f" Files: {p.files_per_partition} | Size: {p.file_size}")
 print(f" Compaction: {p.compaction}")

# Lifecycle Management
@dataclass
class StorageTier:
 tier: str
 age: str
 storage: str
 cost_gb: float
 access: str

tiers = [
 StorageTier("Hot", "0-30 days", "S3 Standard", 0.023, "Frequent queries"),
 StorageTier("Warm", "30-90 days", "S3 Infrequent Access", 0.0125, "Weekly queries"),
 StorageTier("Cold", "90-365 days", "S3 Glacier Instant", 0.004, "Monthly queries"),
 StorageTier("Archive", "1-7 years", "S3 Glacier Deep", 0.00099, "Compliance only"),
]

print(f"\n\n=== Storage Tiers ===")
for t in tiers:
 print(f" [{t.tier}] Age: {t.age} | Storage: {t.storage}")
 print(f" Cost: /GB/month | Access: {t.access}")

เคล็ดลับ

• ZSTD: ใช้ ZSTD level 3 เป็น Default ดีกว่า Snappy ทั้ง Compression และ Speed
• Dictionary: เปิด Dictionary Encoding สำหรับ Column ที่มี Cardinality ต่ำ
• Compact: รวมไฟล์เล็กให้เป็น 128 MB - 1 GB ทุกวัน
• Partition: อย่า Over-partition ไม่เกิน 10000 Partitions
• Statistics: เปิด write_statistics สำหรับ Predicate Pushdown

Parquet Format คืออะไร

Apache Parquet Columnar Storage Column-oriented I/O Compression Snappy Gzip ZSTD LZ4 Predicate Pushdown Schema Spark Hive Presto DuckDB Pandas

สรุป

Parquet Format Capacity Planning Columnar Storage ZSTD Compression Row Group Partition Strategy Data Lake S3 Lifecycle Hot Warm Cold Production