CircleCI Orbs Log Management ELK

CircleCI + ELK

CircleCI Orbs Log Management ELK Elasticsearch Logstash Kibana CI/CD Pipeline Build Logs Dashboard Alert Performance Analysis Production

Orb Configuration

# === CircleCI Config with Orbs ===

# .circleci/config.yml
# version: 2.1
#
# orbs:
#   node: circleci/node@5.1.0
#   docker: circleci/docker@2.4.0
#   aws-cli: circleci/aws-cli@4.1.0
#   elk-logger: my-org/elk-logger@1.0.0
#
# workflows:
#   build-test-deploy:
#     jobs:
#       - node/test:
#           name: unit-tests
#           pkg-manager: yarn
#           post-steps:
#             - elk-logger/send-build-log:
#                 es_host: 
#                 index: circleci-builds
#
#       - docker/publish:
#           name: build-image
#           image: my-app
#           tag: 
#           requires: [unit-tests]
#
#       - deploy:
#           name: deploy-staging
#           requires: [build-image]
#           filters:
#             branches: {only: [main]}

# Custom Orb — elk-logger
# src/commands/send-build-log.yml:
# parameters:
#   es_host:
#     type: string
#   index:
#     type: string
#     default: circleci-builds
# steps:
#   - run:
#       name: Send build log to ELK
#       command: |
#         curl -X POST "//_doc" \
#           -H "Content-Type: application/json" \
#           -d '{
#             "project": "''",
#             "branch": "''",
#             "build_num": '',
#             "status": "''",
#             "sha": "''",
#             "duration_sec": '$(( $(date +%s) -  ))',
#             "timestamp": "'$(date -u +%Y-%m-%dT%H:%M:%SZ)'"
#           }'

from dataclasses import dataclass

@dataclass
class OrbConfig:
    orb_name: str
    version: str
    category: str
    commands: str
    use_case: str

orbs = [
    OrbConfig("circleci/node", "5.1.0", "Language", "install test build", "Node.js projects"),
    OrbConfig("circleci/docker", "2.4.0", "Build", "build publish", "Container images"),
    OrbConfig("circleci/aws-cli", "4.1.0", "Cloud", "setup assume-role", "AWS deployment"),
    OrbConfig("circleci/kubernetes", "1.3.0", "Deploy", "install kubectl rollout", "K8s deployment"),
    OrbConfig("my-org/elk-logger", "1.0.0", "Logging", "send-build-log send-test-log", "ELK integration"),
    OrbConfig("circleci/slack", "4.12.0", "Notify", "notify on-hold", "Slack alerts"),
]

print("=== Orb Registry ===")
for o in orbs:
    print(f"  [{o.orb_name}@{o.version}] {o.category}")
    print(f"    Commands: {o.commands}")
    print(f"    Use: {o.use_case}")

ELK Log Pipeline

# === ELK Pipeline for CI/CD Logs ===

# Logstash Pipeline
# input {
#   http {
#     port => 8080
#     codec => json
#   }
#   # CircleCI Webhook
#   http {
#     port => 8081
#     codec => json
#     additional_codecs => { "application/json" => "json" }
#   }
# }
# filter {
#   mutate {
#     add_field => { "source" => "circleci" }
#   }
#   date {
#     match => ["timestamp", "ISO8601"]
#     target => "@timestamp"
#   }
#   if [duration_sec] {
#     ruby {
#       code => "event.set('duration_min', event.get('duration_sec').to_f / 60)"
#     }
#   }
#   if [status] == "success" {
#     mutate { add_field => { "build_result" => "pass" } }
#   } else {
#     mutate { add_field => { "build_result" => "fail" } }
#   }
# }
# output {
#   elasticsearch {
#     hosts => ["elasticsearch:9200"]
#     index => "circleci-builds-%{+YYYY.MM}"
#   }
# }

# Elasticsearch Index Template
# PUT _index_template/circleci-builds
# {
#   "index_patterns": ["circleci-builds-*"],
#   "template": {
#     "mappings": {
#       "properties": {
#         "project": { "type": "keyword" },
#         "branch": { "type": "keyword" },
#         "build_num": { "type": "integer" },
#         "status": { "type": "keyword" },
#         "build_result": { "type": "keyword" },
#         "duration_sec": { "type": "float" },
#         "duration_min": { "type": "float" },
#         "sha": { "type": "keyword" },
#         "timestamp": { "type": "date" }
#       }
#     }
#   }
# }

@dataclass
class LogField:
    field: str
    es_type: str
    source: str
    dashboard_use: str

fields = [
    LogField("project", "keyword", "CIRCLE_PROJECT_REPONAME", "Filter by project"),
    LogField("branch", "keyword", "CIRCLE_BRANCH", "Filter by branch"),
    LogField("build_num", "integer", "CIRCLE_BUILD_NUM", "Build timeline"),
    LogField("status", "keyword", "Job exit code", "Success/Fail pie chart"),
    LogField("duration_sec", "float", "Calculated", "Duration histogram"),
    LogField("sha", "keyword", "CIRCLE_SHA1", "Link to commit"),
    LogField("timestamp", "date", "ISO8601", "Time series charts"),
    LogField("runner_type", "keyword", "Executor config", "Resource usage"),
]

print("\n=== Elasticsearch Fields ===")
for f in fields:
    print(f"  [{f.field}] Type: {f.es_type}")
    print(f"    Source: {f.source} | Dashboard: {f.dashboard_use}")

Dashboard and Alerts

# === Kibana Dashboard and Alerts ===

@dataclass
class DashboardPanel:
    title: str
    vis_type: str
    query: str
    purpose: str

panels = [
    DashboardPanel("Build Success Rate", "Gauge", "build_result:pass / total * 100",
        "Overall pipeline health indicator"),
    DashboardPanel("Build Duration Trend", "Line Chart", "avg(duration_min) over time",
        "Detect slowing builds"),
    DashboardPanel("Failures by Project", "Bar Chart", "count where build_result:fail group by project",
        "Find problematic projects"),
    DashboardPanel("Branch Activity", "Heat Map", "count group by branch, hour_of_day",
        "Peak development times"),
    DashboardPanel("Top Slow Builds", "Table", "top 10 by duration_sec desc",
        "Optimization targets"),
    DashboardPanel("Deploy Frequency", "Metric", "count where branch:main per day",
        "DORA metric tracking"),
]

print("Kibana Dashboard Panels:")
for p in panels:
    print(f"  [{p.title}] Type: {p.vis_type}")
    print(f"    Query: {p.query}")
    print(f"    Purpose: {p.purpose}")

# Alert Rules
alerts = {
    "Build Fail Spike": "Fail rate > 20% in 1 hour → Slack #ci-alerts",
    "Slow Build": "Duration > 15 min for any project → Slack + Jira",
    "Queue Backup": "Queue time > 5 min for 3 consecutive builds → Scale runners",
    "Flaky Test": "Same test fails > 3 times in 24h → Jira auto-create",
    "Deploy Failure": "Production deploy fails → PagerDuty P2",
    "Daily Digest": "Daily summary: success rate, avg duration, top failures → Email",
}

print(f"\n\nAlert Rules:")
for k, v in alerts.items():
    print(f"  [{k}]: {v}")

เคล็ดลับ

• Orbs: ใช้ Official Orbs ก่อน สร้าง Custom เมื่อจำเป็น
• Index: แยก Index ตามเดือน ลบ Index เก่าอัตโนมัติ
• Dashboard: สร้าง Dashboard สำหรับทุก Team ดู Metrics ตัวเอง
• Alert: ตั้ง Alert เฉพาะที่ Actionable ไม่ Alert ทุกเรื่อง
• Retention: เก็บ Logs 90 วัน ลบอัตโนมัติด้วย ILM

CircleCI Orbs คืออะไร

Reusable Package CI/CD Commands Jobs Executors Library Orb Registry aws-cli docker node Custom Orb องค์กร Public ลดซ้ำซ้อน มาตรฐาน

สรุป

CircleCI Orbs Log Management ELK Elasticsearch Logstash Kibana CI/CD Build Logs Dashboard Alert Performance Analysis Flaky Tests Production