Distributed Tracing #

# Python: instrumentasi dengan OpenTelemetry
from opentelemetry import trace
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.trace.export import BatchSpanProcessor
from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter
from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor
from opentelemetry.instrumentation.httpx import HTTPXClientInstrumentor
from opentelemetry.instrumentation.sqlalchemy import SQLAlchemyInstrumentor

# Setup provider
provider = TracerProvider()
exporter = OTLPSpanExporter(endpoint="http://otel-collector:4317")
provider.add_span_processor(BatchSpanProcessor(exporter))
trace.set_tracer_provider(provider)

# Auto-instrumentasi framework (tidak perlu modifikasi kode bisnis)
FastAPIInstrumentor.instrument_app(app)    # otomatis trace semua HTTP handler
HTTPXClientInstrumentor.instrument()       # otomatis trace semua outbound HTTP
SQLAlchemyInstrumentor.instrument()        # otomatis trace semua query database

# Manual span untuk business logic yang perlu di-trace
tracer = trace.get_tracer(__name__)

def process_payment(order_id: str, amount: float):
    with tracer.start_as_current_span("process_payment") as span:
        span.set_attribute("order.id", order_id)
        span.set_attribute("payment.amount", amount)
        span.set_attribute("payment.currency", "USD")

        try:
            result = payment_gateway.charge(amount)
            span.set_attribute("payment.status", "success")
            return result
        except PaymentError as e:
            span.set_status(trace.StatusCode.ERROR, str(e))
            span.record_exception(e)
            raise

// Go: OpenTelemetry dengan auto-instrumentasi
import (
    "go.opentelemetry.io/otel"
    "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
)

// Wrap HTTP handler
http.Handle("/api/orders", otelhttp.NewHandler(orderHandler, "handle_order"))

// Manual span
tracer := otel.Tracer("order-service")
ctx, span := tracer.Start(ctx, "fetch_inventory")
defer span.End()
span.SetAttributes(attribute.String("product.id", productID))

# OTel Collector sebagai DaemonSet (satu per node)
apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: otel-collector
  namespace: monitoring
spec:
  template:
    spec:
      containers:
      - name: otel-collector
        image: otel/opentelemetry-collector-contrib:0.90.0
        ports:
        - containerPort: 4317   # gRPC receiver
        - containerPort: 4318   # HTTP receiver
        volumeMounts:
        - name: config
          mountPath: /etc/otelcol/
      volumes:
      - name: config
        configMap:
          name: otel-collector-config

# otel-collector-config.yaml
receivers:
  otlp:
    protocols:
      grpc:
        endpoint: 0.0.0.0:4317
      http:
        endpoint: 0.0.0.0:4318

processors:
  batch:                  # batch sebelum export untuk efisiensi
    timeout: 5s
  memory_limiter:
    limit_mib: 512
  resource:               # tambah metadata Kubernetes ke semua span
    attributes:
    - action: insert
      key: k8s.cluster.name
      value: "production"

exporters:
  otlp:                   # kirim ke Jaeger atau Tempo
    endpoint: "jaeger-collector.monitoring:4317"
    tls:
      insecure: true
  logging:                # debug: print ke stdout
    verbosity: normal

service:
  pipelines:
    traces:
      receivers: [otlp]
      processors: [memory_limiter, batch, resource]
      exporters: [otlp]

from opentelemetry.sdk.trace.sampling import (
    TraceIdRatioBased,
    ParentBased,
    ALWAYS_ON
)

# Head-based sampling: keputusan dibuat saat request dimulai
# Simpan 10% dari semua trace
sampler = ParentBased(root=TraceIdRatioBased(0.1))

# Tail-based sampling (di OTel Collector): keputusan dibuat setelah selesai
# Selalu simpan trace dengan error, sample sisanya
# Konfigurasi di OTel Collector:

# Tail-based sampling di OTel Collector
processors:
  tail_sampling:
    decision_wait: 10s        # tunggu 10 detik sebelum memutuskan
    policies:
    - name: error-traces
      type: status_code
      status_code: {status_codes: [ERROR]}   # selalu simpan error
    - name: slow-traces
      type: latency
      latency: {threshold_ms: 1000}          # selalu simpan yang > 1 detik
    - name: rate-limiting
      type: rate_limiting
      rate_limiting: {spans_per_second: 100} # max 100 spans/detik untuk sisanya