Databricks Unity Catalog Community Building

Databricks Unity Catalog Community Building คืออะไร

Databricks Unity Catalog เป็น unified governance solution สำหรับ data lakehouse ที่จัดการ data, ML models, notebooks และ files ทั้งหมดในที่เดียว Community Building คือการสร้างและพัฒนา community ของผู้ใช้งาน data platform ภายในองค์กร เพื่อส่งเสริมการแชร์ความรู้ best practices และ collaboration ระหว่างทีม data การรวม Unity Catalog กับ community building ช่วยให้องค์กรได้ประโยชน์สูงสุดจาก data assets พร้อม Python tools สำหรับ automation

Unity Catalog Architecture

# unity_catalog.py — Unity Catalog architecture
import json

class UnityCatalogArch:
    HIERARCHY = {
        "metastore": {
            "name": "Metastore",
            "description": "Top-level container — 1 per region, จัดการ metadata ทั้งหมด",
        },
        "catalog": {
            "name": "Catalog",
            "description": "Namespace level 1 — แบ่งตาม environment (dev/staging/prod) หรือ domain",
        },
        "schema": {
            "name": "Schema (Database)",
            "description": "Namespace level 2 — จัดกลุ่ม tables, views, functions",
        },
        "objects": {
            "name": "Data Objects",
            "description": "Tables, Views, Volumes, Functions, Models — managed assets",
        },
    }

    FEATURES = {
        "centralized_governance": {
            "name": "Centralized Governance",
            "description": "จัดการ permissions, audit, lineage จากที่เดียว",
        },
        "data_lineage": {
            "name": "Data Lineage",
            "description": "ติดตาม data flow — รู้ว่า data มาจากไหน ไปถึงไหน",
        },
        "access_control": {
            "name": "Fine-grained Access Control",
            "description": "GRANT/REVOKE permissions ระดับ table, column, row",
        },
        "data_sharing": {
            "name": "Delta Sharing",
            "description": "แชร์ data ข้าม organizations — open protocol",
        },
        "ai_models": {
            "name": "ML Model Registry",
            "description": "จัดการ ML models — versioning, staging, production",
        },
    }

    def show_hierarchy(self):
        print("=== Unity Catalog Hierarchy ===\n")
        indent = ""
        for key, level in self.HIERARCHY.items():
            print(f"{indent}[{level['name']}]")
            print(f"{indent}  {level['description']}")
            indent += "  "

    def show_features(self):
        print(f"\n=== Key Features ===")
        for key, feat in self.FEATURES.items():
            print(f"  [{feat['name']}] {feat['description']}")

arch = UnityCatalogArch()
arch.show_hierarchy()
arch.show_features()

Community Building Framework

# community.py — Community building framework
import json

class CommunityFramework:
    PILLARS = {
        "knowledge_sharing": {
            "name": "1. Knowledge Sharing",
            "activities": [
                "Monthly Data Community Meetup — แชร์ use cases, best practices",
                "Internal Wiki/Confluence — documentation, tutorials, FAQs",
                "Slack/Teams channels — #data-help, #unity-catalog, #sql-tips",
                "Lunch & Learn sessions — 30 นาที demo + Q&A",
            ],
        },
        "standards": {
            "name": "2. Standards & Best Practices",
            "activities": [
                "Naming conventions — catalog.schema.table format",
                "Data quality standards — expectations, validation rules",
                "Access control policies — RBAC templates per team",
                "Documentation requirements — README per schema",
            ],
        },
        "enablement": {
            "name": "3. Self-Service Enablement",
            "activities": [
                "Data catalog UI — search, discover, request access",
                "Template notebooks — common patterns, starter code",
                "Training program — onboarding, advanced workshops",
                "Office hours — weekly drop-in sessions with data team",
            ],
        },
        "governance": {
            "name": "4. Governance & Compliance",
            "activities": [
                "Data stewardship — assign owners per catalog/schema",
                "Access review — quarterly review of permissions",
                "Audit logging — track who accessed what data",
                "PII/sensitive data tagging — classification labels",
            ],
        },
    }

    METRICS = {
        "adoption": "% ของทีมที่ใช้ Unity Catalog (target: > 80%)",
        "discovery": "จำนวน data assets ที่ documented + discoverable",
        "sharing": "จำนวน datasets ที่แชร์ข้าม teams",
        "quality": "% ของ tables ที่ pass quality checks",
        "engagement": "จำนวนผู้เข้าร่วม community events",
    }

    def show_pillars(self):
        print("=== Community Building Pillars ===\n")
        for key, pillar in self.PILLARS.items():
            print(f"[{pillar['name']}]")
            for activity in pillar['activities'][:2]:
                print(f"  • {activity}")
            print()

    def show_metrics(self):
        print("=== Success Metrics ===")
        for metric, desc in self.METRICS.items():
            print(f"  [{metric}] {desc}")

framework = CommunityFramework()
framework.show_pillars()
framework.show_metrics()

Python Unity Catalog Tools

# uc_tools.py — Python tools for Unity Catalog
import json

class UCTools:
    CODE = """
# unity_catalog_tools.py — Automate Unity Catalog management
import requests
import json
from datetime import datetime

class UnityCatalogManager:
    def __init__(self, workspace_url, token):
        self.base_url = f"{workspace_url}/api/2.1/unity-catalog"
        self.headers = {"Authorization": f"Bearer {token}"}
    
    def list_catalogs(self):
        '''List all catalogs'''
        resp = requests.get(f"{self.base_url}/catalogs", headers=self.headers)
        return resp.json().get('catalogs', [])
    
    def list_schemas(self, catalog_name):
        '''List schemas in a catalog'''
        resp = requests.get(
            f"{self.base_url}/schemas",
            params={"catalog_name": catalog_name},
            headers=self.headers,
        )
        return resp.json().get('schemas', [])
    
    def list_tables(self, catalog_name, schema_name):
        '''List tables in a schema'''
        resp = requests.get(
            f"{self.base_url}/tables",
            params={
                "catalog_name": catalog_name,
                "schema_name": schema_name,
            },
            headers=self.headers,
        )
        return resp.json().get('tables', [])
    
    def get_table_info(self, full_name):
        '''Get table details'''
        resp = requests.get(
            f"{self.base_url}/tables/{full_name}",
            headers=self.headers,
        )
        return resp.json()
    
    def grant_permissions(self, securable_type, full_name, principal, privileges):
        '''Grant permissions on a securable'''
        resp = requests.patch(
            f"{self.base_url}/permissions/{securable_type}/{full_name}",
            headers=self.headers,
            json={
                "changes": [{
                    "principal": principal,
                    "add": privileges,
                }]
            },
        )
        return resp.json()
    
    def catalog_inventory(self):
        '''Generate full catalog inventory'''
        inventory = []
        
        for catalog in self.list_catalogs():
            cat_name = catalog['name']
            
            for schema in self.list_schemas(cat_name):
                sch_name = schema['name']
                
                for table in self.list_tables(cat_name, sch_name):
                    inventory.append({
                        'catalog': cat_name,
                        'schema': sch_name,
                        'table': table['name'],
                        'type': table.get('table_type', 'UNKNOWN'),
                        'owner': table.get('owner', ''),
                        'comment': table.get('comment', ''),
                        'created': table.get('created_at', ''),
                    })
        
        return {
            'timestamp': datetime.utcnow().isoformat(),
            'total_assets': len(inventory),
            'assets': inventory,
        }
    
    def documentation_report(self):
        '''Check documentation coverage'''
        inventory = self.catalog_inventory()
        
        documented = sum(1 for a in inventory['assets'] if a['comment'])
        owned = sum(1 for a in inventory['assets'] if a['owner'])
        
        return {
            'total_tables': inventory['total_assets'],
            'documented': documented,
            'documentation_pct': round(documented / max(inventory['total_assets'], 1) * 100, 1),
            'owned': owned,
            'ownership_pct': round(owned / max(inventory['total_assets'], 1) * 100, 1),
            'undocumented': [
                a for a in inventory['assets'] if not a['comment']
            ],
        }

# mgr = UnityCatalogManager("https://workspace.cloud.databricks.com", "dapi-xxx")
# report = mgr.documentation_report()
"""

    def show_code(self):
        print("=== Unity Catalog Tools ===")
        print(self.CODE[:600])

tools = UCTools()
tools.show_code()

Community Events & Programs

# events.py — Community events and programs
import json

class CommunityEvents:
    PROGRAMS = {
        "data_champions": {
            "name": "Data Champions Program",
            "description": "เลือกตัวแทนจากแต่ละทีม — เป็น ambassador ของ data platform",
            "responsibilities": [
                "ช่วยทีมใช้ Unity Catalog อย่างถูกต้อง",
                "รวบรวม feedback และ feature requests",
                "แชร์ best practices ในทีม",
                "เข้าร่วมประชุม monthly champions meetup",
            ],
            "benefits": "Recognition, training budget, conference tickets",
        },
        "hackathon": {
            "name": "Data Hackathon (Quarterly)",
            "description": "แข่งขันสร้าง data products ภายใน 1-2 วัน",
            "themes": [
                "Data quality improvement — ทำ tables ให้มี quality checks ครบ",
                "Dashboard challenge — สร้าง dashboard ที่มีประโยชน์ที่สุด",
                "ML use case — สร้าง ML model จาก Unity Catalog data",
                "Documentation marathon — document undocumented tables",
            ],
        },
        "onboarding": {
            "name": "Data Platform Onboarding",
            "description": "โปรแกรม onboarding สำหรับพนักงานใหม่",
            "modules": [
                "Week 1: Unity Catalog basics — catalog/schema/table hierarchy",
                "Week 2: SQL + Spark basics — query, transform, write",
                "Week 3: Access control — request access, understand RBAC",
                "Week 4: Best practices — naming, documentation, quality",
            ],
        },
    }

    def show_programs(self):
        print("=== Community Programs ===\n")
        for key, prog in self.PROGRAMS.items():
            print(f"[{prog['name']}]")
            print(f"  {prog['description']}")
            if 'responsibilities' in prog:
                for r in prog['responsibilities'][:2]:
                    print(f"    • {r}")
            elif 'themes' in prog:
                for t in prog['themes'][:2]:
                    print(f"    • {t}")
            elif 'modules' in prog:
                for m in prog['modules'][:2]:
                    print(f"    • {m}")
            print()

events = CommunityEvents()
events.show_programs()

Automation & Monitoring

# automation.py — Automated community health monitoring
import json
import random

class CommunityMonitoring:
    AUTOMATIONS = {
        "weekly_report": {
            "name": "Weekly Data Platform Report",
            "description": "สรุปสถิติการใช้งาน Unity Catalog ทุกสัปดาห์",
            "metrics": ["New tables created", "Queries executed", "Active users", "Documentation coverage"],
        },
        "stale_data_alert": {
            "name": "Stale Data Alert",
            "description": "แจ้งเตือนเมื่อ table ไม่ได้ update > 7 วัน",
        },
        "access_review": {
            "name": "Quarterly Access Review",
            "description": "ส่ง email ให้ data owners review permissions ทุก quarter",
        },
        "onboarding_bot": {
            "name": "Onboarding Slack Bot",
            "description": "Bot ตอบคำถามพื้นฐาน + guide ผ่าน onboarding steps",
        },
    }

    def show_automations(self):
        print("=== Community Automations ===\n")
        for key, auto in self.AUTOMATIONS.items():
            print(f"[{auto['name']}]")
            print(f"  {auto['description']}")

    def sample_dashboard(self):
        print(f"\n=== Community Health Dashboard ===")
        print(f"  Active Users (7d): {random.randint(50, 200)}")
        print(f"  Tables Created (7d): {random.randint(10, 50)}")
        print(f"  Queries (7d): {random.randint(5000, 20000)}")
        print(f"  Documentation Coverage: {random.uniform(60, 95):.1f}%")
        print(f"  Avg Query Time: {random.uniform(2, 15):.1f}s")
        print(f"  Community Events (month): {random.randint(2, 8)}")
        print(f"  Support Tickets: {random.randint(5, 30)}")

monitor = CommunityMonitoring()
monitor.show_automations()
monitor.sample_dashboard()

FAQ - คำถามที่พบบ่อย

Q: Unity Catalog จำเป็นต้องใช้ไหม?

A: แนะนำอย่างยิ่งถ้าใช้ Databricks — เป็น default governance layer ข้อดี: centralized permissions, lineage, data discovery, Delta Sharing ถ้าไม่ใช้: ต้องจัดการ permissions แยกต่างหาก, ไม่มี lineage, data discovery ยาก ข้อจำกัด: ต้องเป็น Databricks Premium/Enterprise plan

Q: Community building สำคัญจริงหรือ?

A: สำคัญมาก — data platform ที่ดีแต่ไม่มีคนใช้ = เสียเงินเปล่า ปัญหาที่พบ: teams ไม่รู้ว่ามี data อะไรบ้าง, duplicate data, ไม่มี standards Community ช่วย: knowledge sharing, adoption, data quality, collaboration ROI: ลด duplicated work 30-50%, เพิ่ม data discovery, ลด support tickets

Q: เริ่มสร้าง data community อย่างไร?

A: 1) หา executive sponsor — ต้องมี support จาก management 2) สร้าง core team 3-5 คน — data champions จากแต่ละทีม 3) เริ่มจาก Slack channel + monthly meetup 4) สร้าง documentation + templates 5) วัดผลด้วย metrics — adoption, engagement, data quality สำคัญ: เริ่มเล็กๆ แล้วค่อยขยาย — อย่าพยายามทำทุกอย่างพร้อมกัน

Q: Delta Sharing คืออะไร?

A: Open protocol สำหรับแชร์ data ข้าม organizations อย่างปลอดภัย ข้อดี: ไม่ต้อง copy data, receiver ใช้ tool อะไรก็ได้ (Pandas, Spark, Power BI) ใช้งาน: สร้าง Share → เพิ่ม tables → สร้าง Recipient → แชร์ credentials เหมาะ: B2B data sharing, partner data exchange, open data publishing