TensorFlow Serving Best Practices ที่ต้องรู้ — วิธี Deploy ML Models ด้วย TensorFlow Serving อย่างมืออาชีพ | SiamCafe Blog

TensorFlow Serving คืออะไร

TensorFlow Serving เป็นระบบ High-performance Serving สำหรับ Machine Learning Models ที่ Google พัฒนา ออกแบบมาสำหรับ Production Environment รองรับทั้ง TensorFlow SavedModel และ TensorFlow Lite Models ให้บริการผ่าน gRPC และ REST API

จุดเด่นคือ Performance สูง (C++ Backend), Model Versioning อัตโนมัติ, Batching ที่ Optimize สำหรับ GPU, Hot-reload Models ใหม่โดยไม่ต้อง Restart Service และรองรับ Multiple Models พร้อมกัน

Installation และ Configuration

# === TensorFlow Serving Setup ===

# 1. Docker (วิธีที่ง่ายที่สุด)
docker pull tensorflow/serving:latest
docker pull tensorflow/serving:latest-gpu # สำหรับ GPU

# 2. โครงสร้าง Model Directory
# /models/
# ├── text_classifier/
# │ ├── 1/ # Version 1
# │ │ ├── saved_model.pb
# │ │ └── variables/
# │ ├── 2/ # Version 2
# │ │ ├── saved_model.pb
# │ │ └── variables/
# │ └── 3/ # Version 3 (latest)
# │ ├── saved_model.pb
# │ └── variables/
# ├── image_classifier/
# │ └── 1/
# │ ├── saved_model.pb
# │ └── variables/
# └── recommender/
# └── 1/
# ├── saved_model.pb
# └── variables/

# 3. รันด้วย Docker (Single Model)
docker run -d --name tf-serving \
 -p 8501:8501 \
 -p 8500:8500 \
 -v /path/to/models/text_classifier:/models/text_classifier \
 -e MODEL_NAME=text_classifier \
 tensorflow/serving

# 4. รันด้วย Docker (Multiple Models + Config)
docker run -d --name tf-serving-multi \
 -p 8501:8501 \
 -p 8500:8500 \
 -v /path/to/models:/models \
 -v /path/to/model_config.txt:/config/model_config.txt \
 tensorflow/serving \
 --model_config_file=/config/model_config.txt \
 --model_config_file_poll_wait_seconds=60

# 5. Model Config File (model_config.txt)
# model_config_list {
# config {
# name: "text_classifier"
# base_path: "/models/text_classifier"
# model_platform: "tensorflow"
# model_version_policy {
# specific {
# versions: 2
# versions: 3
# }
# }
# }
# config {
# name: "image_classifier"
# base_path: "/models/image_classifier"
# model_platform: "tensorflow"
# }
# config {
# name: "recommender"
# base_path: "/models/recommender"
# model_platform: "tensorflow"
# }
# }

# 6. Batching Config (batching_parameters.txt)
# max_batch_size { value: 32 }
# batch_timeout_micros { value: 5000 }
# max_enqueued_batches { value: 100 }
# num_batch_threads { value: 4 }
# pad_variable_length_inputs: true

# 7. รันพร้อม Batching
docker run -d --name tf-serving-batch \
 -p 8501:8501 \
 -p 8500:8500 \
 -v /path/to/models:/models \
 -v /path/to/model_config.txt:/config/model_config.txt \
 -v /path/to/batching_parameters.txt:/config/batching.txt \
 tensorflow/serving \
 --model_config_file=/config/model_config.txt \
 --enable_batching=true \
 --batching_parameters_file=/config/batching.txt

# 8. GPU Support
docker run -d --name tf-serving-gpu \
 --gpus all \
 -p 8501:8501 \
 -p 8500:8500 \
 -v /path/to/models:/models \
 -e MODEL_NAME=text_classifier \
 tensorflow/serving:latest-gpu \
 --per_process_gpu_memory_fraction=0.5

# 9. ตรวจสอบ Model Status
curl http://localhost:8501/v1/models/text_classifier
curl http://localhost:8501/v1/models/text_classifier/versions/3

Python Client สำหรับ TensorFlow Serving

# tf_serving_client.py — Client สำหรับ TensorFlow Serving
# pip install tensorflow-serving-api grpcio requests numpy

import requests
import numpy as np
import json
import time
import grpc
from concurrent import futures
import logging

logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)

class TFServingClient:
 """Client สำหรับ TensorFlow Serving"""

 def __init__(self, host="localhost", rest_port=8501, grpc_port=8500):
 self.rest_url = f"http://{host}:{rest_port}"
 self.grpc_target = f"{host}:{grpc_port}"

 # === REST API ===
 def predict_rest(self, model_name, instances, version=None):
 """Predict ผ่าน REST API"""
 url = f"{self.rest_url}/v1/models/{model_name}"
 if version:
 url += f"/versions/{version}"
 url += ":predict"

 payload = {"instances": instances}

 start = time.perf_counter()
 resp = requests.post(url, json=payload, timeout=30)
 latency = (time.perf_counter() - start) * 1000

 if resp.status_code != 200:
 logger.error(f"REST Error: {resp.status_code} {resp.text}")
 raise Exception(f"Prediction failed: {resp.text}")

 result = resp.json()
 result["_latency_ms"] = round(latency, 2)
 return result

 # === gRPC API (faster) ===
 def predict_grpc(self, model_name, input_data, input_name="input",
 version=None):
 """Predict ผ่าน gRPC (เร็วกว่า REST)"""
 from tensorflow_serving.apis import predict_pb2
 from tensorflow_serving.apis import prediction_service_pb2_grpc
 import tensorflow as tf

 channel = grpc.insecure_channel(self.grpc_target)
 stub = prediction_service_pb2_grpc.PredictionServiceStub(channel)

 request = predict_pb2.PredictRequest()
 request.model_spec.name = model_name
 if version:
 request.model_spec.version.value = version

 request.inputs[input_name].CopyFrom(
 tf.make_tensor_proto(input_data)
 )

 start = time.perf_counter()
 response = stub.Predict(request, timeout=30)
 latency = (time.perf_counter() - start) * 1000

 output = {}
 for key, tensor in response.outputs.items():
 output[key] = tf.make_ndarray(tensor).tolist()
 output["_latency_ms"] = round(latency, 2)

 return output

 def get_model_status(self, model_name):
 """ดูสถานะ Model"""
 url = f"{self.rest_url}/v1/models/{model_name}"
 resp = requests.get(url)
 return resp.json()

 def get_model_metadata(self, model_name):
 """ดู Model Metadata (input/output signature)"""
 url = f"{self.rest_url}/v1/models/{model_name}/metadata"
 resp = requests.get(url)
 return resp.json()

 def benchmark(self, model_name, instances, n_requests=100):
 """Benchmark Prediction Performance"""
 latencies = []

 # Warmup
 for _ in range(5):
 self.predict_rest(model_name, instances)

 # Benchmark
 for i in range(n_requests):
 result = self.predict_rest(model_name, instances)
 latencies.append(result["_latency_ms"])

 avg = np.mean(latencies)
 p50 = np.percentile(latencies, 50)
 p95 = np.percentile(latencies, 95)
 p99 = np.percentile(latencies, 99)
 qps = 1000 / avg * len(instances)

 print(f"\n{'='*50}")
 print(f"TF Serving Benchmark: {model_name}")
 print(f"{'='*50}")
 print(f" Requests: {n_requests}")
 print(f" Batch: {len(instances)}")
 print(f" Avg: {avg:.1f}ms")
 print(f" P50: {p50:.1f}ms")
 print(f" P95: {p95:.1f}ms")
 print(f" P99: {p99:.1f}ms")
 print(f" QPS: {qps:.0f}")

 return {"avg": avg, "p50": p50, "p95": p95, "p99": p99, "qps": qps}

# === ตัวอย่าง ===
# client = TFServingClient("localhost", 8501, 8500)
#
# # REST Prediction
# result = client.predict_rest("text_classifier", [
# {"input_text": "This is a great product"},
# {"input_text": "Terrible service"},
# ])
# print(result)
#
# # Model Status
# status = client.get_model_status("text_classifier")
# print(status)
#
# # Benchmark
# client.benchmark("text_classifier", [{"input_text": "test"}], n_requests=100)

Kubernetes Deployment

# === Kubernetes Deployment สำหรับ TensorFlow Serving ===
# tf-serving-deployment.yaml

apiVersion: apps/v1
kind: Deployment
metadata:
 name: tf-serving
 namespace: ml-serving
spec:
 replicas: 3
 selector:
 matchLabels:
 app: tf-serving
 template:
 metadata:
 labels:
 app: tf-serving
 spec:
 affinity:
 podAntiAffinity:
 preferredDuringSchedulingIgnoredDuringExecution:
 - weight: 100
 podAffinityTerm:
 labelSelector:
 matchExpressions:
 - key: app
 operator: In
 values: ["tf-serving"]
 topologyKey: kubernetes.io/hostname
 containers:
 - name: tf-serving
 image: tensorflow/serving:latest
 ports:
 - containerPort: 8501
 name: rest
 - containerPort: 8500
 name: grpc
 args:
 - --model_config_file=/config/model_config.txt
 - --enable_batching=true
 - --batching_parameters_file=/config/batching.txt
 - --monitoring_config_file=/config/monitoring.txt
 - --rest_api_num_threads=16
 resources:
 requests:
 cpu: "2"
 memory: 4Gi
 limits:
 cpu: "4"
 memory: 8Gi
 volumeMounts:
 - name: models
 mountPath: /models
 readOnly: true
 - name: config
 mountPath: /config
 readinessProbe:
 httpGet:
 path: /v1/models/text_classifier
 port: 8501
 initialDelaySeconds: 30
 periodSeconds: 10
 livenessProbe:
 httpGet:
 path: /v1/models/text_classifier
 port: 8501
 initialDelaySeconds: 60
 periodSeconds: 30
 volumes:
 - name: models
 persistentVolumeClaim:
 claimName: model-storage
 - name: config
 configMap:
 name: tf-serving-config
---
apiVersion: v1
kind: Service
metadata:
 name: tf-serving
 namespace: ml-serving
spec:
 selector:
 app: tf-serving
 ports:
 - name: rest
 port: 8501
 targetPort: 8501
 - name: grpc
 port: 8500
 targetPort: 8500
---
apiVersion: autoscaling/v2
kind: HorizontalPodAutoscaler
metadata:
 name: tf-serving-hpa
 namespace: ml-serving
spec:
 scaleTargetRef:
 apiVersion: apps/v1
 kind: Deployment
 name: tf-serving
 minReplicas: 3
 maxReplicas: 20
 metrics:
 - type: Resource
 resource:
 name: cpu
 target:
 type: Utilization
 averageUtilization: 60

Best Practices

• ใช้ gRPC สำหรับ Low Latency: gRPC เร็วกว่า REST 2-3 เท่า เหมาะกับ Internal Services
• เปิด Batching: รวม Requests เป็น Batch ลด GPU Overhead เพิ่ม Throughput
• Model Versioning: ใช้ Version Directories Auto-detect Version ใหม่ ทำ Canary Deployment
• Resource Limits: ตั้ง CPU/Memory Limits เหมาะสม ใช้ per_process_gpu_memory_fraction สำหรับ GPU
• Health Checks: ใช้ /v1/models/MODEL_NAME สำหรับ Readiness Probe
• Monitoring: ติดตาม Latency, Throughput, Error Rate, GPU Utilization ตั้ง Alert
• Warm-up: ใช้ TF Serving Warmup Feature ส่ง Sample Requests ตอน Start ลด Cold Start

TensorFlow Serving คืออะไร

ระบบ High-performance Serving สำหรับ ML Models จาก Google รองรับ gRPC REST API Model Versioning Batching GPU Acceleration Hot-reload Models ไม่ต้อง Restart