可观测性

三大支柱

可观测性 = 日志 (Logging) + 指标 (Metrics) + 链路追踪 (Tracing)

支柱	回答的问题	工具
日志	发生了什么？为什么出错？	slog / zap / ELK
指标	系统状态如何？趋势变化？	Prometheus / OpenTelemetry
链路追踪	请求经过了哪些服务？哪一步慢？	Jaeger / SkyWalking

日志（Logging）

结构化日志中间件

func Logging(logger *slog.Logger) func(http.Handler) http.Handler {
    return func(next http.Handler) http.Handler {
        return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
            start := time.Now()

            // 包装 ResponseWriter 记录状态码
            rec := &statusRecorder{ResponseWriter: w, status: 200}

            // 提取 trace ID
            traceID := r.Header.Get("X-Trace-ID")
            if traceID == "" {
                traceID = generateTraceID()
                w.Header().Set("X-Trace-ID", traceID)
            }

            next.ServeHTTP(rec, r)

            // 记录日志
            logger.Info("request",
                "method", r.Method,
                "path", r.URL.Path,
                "status", rec.status,
                "size", rec.size,
                "duration", time.Since(start).String(),
                "ip", clientIP(r),
                "user_agent", r.UserAgent(),
                "trace_id", traceID,
            )
        })
    }
}

日志级别使用指南

级别	使用场景	示例
Debug	开发调试信息	”cache miss for key: xxx”
Info	正常业务事件	”user login”, “order created”
Warn	异常但不影响功能	”retry connection”, “slow query”
Error	错误需要关注	”db connection failed”, “payment failed”
Fatal	不可恢复，程序退出	”config load failed”

指标（Metrics）

Prometheus 指标

import "github.com/prometheus/client_golang/prometheus"
import "github.com/prometheus/client_golang/prometheus/promhttp"

// Counter：只增不减（请求数、错误数）
var httpRequestsTotal = prometheus.NewCounterVec(
    prometheus.CounterOpts{
        Name: "http_requests_total",
        Help: "Total number of HTTP requests",
    },
    []string{"method", "path", "status"},
)

// Gauge：可增可减（当前连接数、队列长度）
var activeConnections = prometheus.NewGauge(
    prometheus.GaugeOpts{
        Name: "active_connections",
        Help: "Number of active connections",
    },
)

// Histogram：分布统计（响应时间分布）
var httpDuration = prometheus.NewHistogramVec(
    prometheus.HistogramOpts{
        Name:    "http_request_duration_seconds",
        Help:    "HTTP request duration in seconds",
        Buckets: []float64{0.001, 0.01, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10},
    },
    []string{"method", "path"},
)

// Summary：分位数统计（p99 延迟）
var httpDurationSummary = prometheus.NewSummaryVec(
    prometheus.SummaryOpts{
        Name:       "http_request_duration_summary",
        Help:       "HTTP request duration summary",
        Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001},
    },
    []string{"method", "path"},
)

func init() {
    prometheus.MustRegister(
        httpRequestsTotal,
        activeConnections,
        httpDuration,
        httpDurationSummary,
    )
}

指标中间件

func Metrics() func(http.Handler) http.Handler {
    return func(next http.Handler) http.Handler {
        return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
            start := time.Now()
            rec := &statusRecorder{ResponseWriter: w, status: 200}

            next.ServeHTTP(rec, r)

            duration := time.Since(start).Seconds()

            httpRequestsTotal.WithLabelValues(r.Method, r.URL.Path, strconv.Itoa(rec.status)).Inc()
            httpDuration.WithLabelValues(r.Method, r.URL.Path).Observe(duration)
        })
    }
}

// 注册 Prometheus 端点
mux.Handle("/metrics", promhttp.Handler())

指标类型选择

类型	适用	示例
Counter	累计值	请求总数、错误总数、已处理字节数
Gauge	瞬时值	当前连接数、内存使用、队列长度
Histogram	分布	响应时间、请求大小
Summary	分位数	P99 延迟（不如 Histogram 可聚合）

业务指标示例

// 业务指标
var (
    ordersCreated = prometheus.NewCounter(
        prometheus.CounterOpts{
            Name: "orders_created_total",
            Help: "Total orders created",
        },
    )
    orderAmount = prometheus.NewHistogram(
        prometheus.HistogramOpts{
            Name:    "order_amount_yuan",
            Help:    "Order amount in yuan",
            Buckets: []float64{10, 50, 100, 500, 1000, 5000, 10000},
        },
    )
    activeUsers = prometheus.NewGauge(
        prometheus.GaugeOpts{
            Name: "active_users",
            Help: "Number of active users",
        },
    )
)

链路追踪（Tracing）

SkyWalking-Go 集成（care-mate 项目）

import skywalking "github.com/SkyAPM/go2sky"

// 初始化 tracer
tracer, err := skywalking.NewTracer("care-mate")
if err != nil {
    log.Fatal(err)
}

// 设置 OAP 服务地址
reporter, err := reporter.NewGRPCReporter("skywalking-oap:11800")
if err != nil {
    log.Fatal(err)
}
tracer.RegisterReporter(reporter)

// HTTP 中间件
import "github.com/SkyAPM/go2sky/plugins/http"

func Tracing(tracer *skywalking.Tracer) func(http.Handler) http.Handler {
    return func(next http.Handler) http.Handler {
        return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
            span, ctx, err := tracer.CreateEntrySpan(r.Context(), fmt.Sprintf("%s %s", r.Method, r.URL.Path))
            if err == nil {
                defer span.End()
                span.Tag("http.method", r.Method)
                span.Tag("http.url", r.URL.String())
                span.Tag("http.remote_addr", r.RemoteAddr)
            }
            next.ServeHTTP(w, r.WithContext(ctx))
        })
    }
}

OpenTelemetry 集成（通用方案）

import (
    "go.opentelemetry.io/otel"
    "go.opentelemetry.io/otel/exporters/jaeger"
    "go.opentelemetry.io/otel/sdk/resource"
    "go.opentelemetry.io/otel/sdk/trace"
    semconv "go.opentelemetry.io/otel/semconv/v1.21.0"
)

func InitTracer(serviceName, jaegerURL string) (*trace.TracerProvider, error) {
    exporter, err := jaeger.New(jaeger.WithCollectorEndpoint(jaeger.WithEndpoint(jaegerURL)))
    if err != nil {
        return nil, err
    }

    tp := trace.NewTracerProvider(
        trace.WithBatcher(exporter),
        trace.WithResource(resource.NewWithAttributes(
            semconv.SchemaURL,
            semconv.ServiceName(serviceName),
            semconv.ServiceVersion("1.0.0"),
        )),
    )

    otel.SetTracerProvider(tp)
    return tp, nil
}

// 使用
tracer := otel.Tracer("care-mate")
ctx, span := tracer.Start(r.Context(), "GetUser")
defer span.End()

span.SetAttributes(
    attribute.String("user.id", userID),
    attribute.String("user.name", user.Name),
)

手动创建 Span

func (s *UserService) GetUser(ctx context.Context, id string) (*User, error) {
    ctx, span := tracer.Start(ctx, "UserService.GetUser")
    defer span.End()

    span.SetAttributes(attribute.String("user.id", id))

    // 从缓存查找
    ctx, cacheSpan := tracer.Start(ctx, "redis.Get")
    cached, err := s.cache.Get(ctx, "user:"+id)
    cacheSpan.End()

    if err == nil {
        span.SetAttributes(attribute.Bool("cache.hit", true))
        return cached, nil
    }

    span.SetAttributes(attribute.Bool("cache.hit", false))

    // 从数据库查找
    ctx, dbSpan := tracer.Start(ctx, "db.Query")
    user, err := s.repo.FindByID(ctx, id)
    dbSpan.End()

    if err != nil {
        span.RecordError(err)
        return nil, err
    }

    // 写入缓存
    s.cache.Set(ctx, "user:"+id, user, 5*time.Minute)
    return user, nil
}

健康检查

// 健康检查端点
func HealthHandler(db *sql.DB, redis *redis.Client) http.HandlerFunc {
    return func(w http.ResponseWriter, r *http.Request) {
        checks := map[string]string{}
        allHealthy := true

        // 检查数据库
        if err := db.PingContext(r.Context()); err != nil {
            checks["database"] = "unhealthy: " + err.Error()
            allHealthy = false
        } else {
            checks["database"] = "healthy"
        }

        // 检查 Redis
        if err := redis.Ping(r.Context()).Err(); err != nil {
            checks["redis"] = "unhealthy: " + err.Error()
            allHealthy = false
        } else {
            checks["redis"] = "healthy"
        }

        status := http.StatusOK
        if !allHealthy {
            status = http.StatusServiceUnavailable
        }

        w.Header().Set("Content-Type", "application/json")
        w.WriteHeader(status)
        json.NewEncoder(w).Encode(map[string]interface{}{
            "status":  map[bool]string{true: "healthy", false: "unhealthy"}[allHealthy],
            "checks":  checks,
            "version": Version,
            "time":    time.Now().UTC(),
        })
    }
}

// K8s 就绪/存活探针
// 就绪探针：检查是否准备好接收流量
mux.HandleFunc("/healthz", HealthHandler(db, redis))  // 存活
mux.HandleFunc("/readyz", ReadyHandler(db, redis))    // 就绪

// K8s 配置
// livenessProbe:
//   httpGet:
//     path: /healthz
//     port: 8080
//   initialDelaySeconds: 10
//   periodSeconds: 30
// readinessProbe:
//   httpGet:
//     path: /readyz
//     port: 8080
//   initialDelaySeconds: 5
//   periodSeconds: 10

可观测性检查清单

检查项	日志	指标	追踪
请求级	✅ 每请求一条日志	✅ 请求数/延迟	✅ 入口 Span
业务级	✅ 关键业务事件	✅ 业务计数器	✅ 操作 Span
错误级	✅ Error 日志	✅ 错误计数	✅ 错误标记
基础设施	✅ 连接日志	✅ 连接池/内存	✅ DB/Redis Span
关联 ID	✅ trace_id	-	✅ 贯穿全链路

生产环境必须的可观测性端点

端点	用途	探针
/healthz	存活检查	K8s liveness
/readyz	就绪检查	K8s readiness
/metrics	Prometheus 指标	Prometheus scrape
/debug/pprof/*	性能分析	仅 localhost