Go Observability & pprof — Memory Leaks, CPU Profiling & GODEBUG

package main

import (
    "log"
    "net/http"
    _ "net/http/pprof" // Side-effect import: registers /debug/pprof/* handlers
)

func main() {
    // pprof server on a separate port — NEVER expose this publicly
    go func() {
        log.Println("pprof listening on localhost:6060")
        // localhost binding ensures it's only accessible from within the host or via port-forward
        if err := http.ListenAndServe("localhost:6060", nil); err != nil {
            log.Printf("pprof server error: %v", err)
        }
    }()

    // Main application server
    http.HandleFunc("/health", func(w http.ResponseWriter, r *http.Request) {
        w.WriteHeader(http.StatusOK)
    })
    log.Fatal(http.ListenAndServe(":8080", nil))
}

# Step 1: Capture baseline heap
curl -sK -v -o baseline.pprof http://localhost:6060/debug/pprof/heap
echo "Baseline captured: $(date)"

# Step 2: Run load or wait 5–15 minutes under production traffic

# Step 3: Capture peak heap snapshot
curl -sK -v -o peak.pprof http://localhost:6060/debug/pprof/heap
echo "Peak captured: $(date)"

# Step 4: Diff profiles — shows ONLY the increase (the leak signal)
go tool pprof -base baseline.pprof peak.pprof

# Step 5: In interactive shell
(pprof) top 20          # Top 20 functions by allocation increase
(pprof) list SuspectFunc # Show per-line allocation detail for a specific function
(pprof) web             # Open flame graph in browser (requires graphviz)

# CPU profile: 30 seconds of sampling (5–10% CPU overhead during capture)
go tool pprof http://localhost:6060/debug/pprof/profile?seconds=30

# Heap profile — focus on live objects (memory leak detection)
go tool pprof -inuse_space http://localhost:6060/debug/pprof/heap

# Heap profile — focus on total allocations (GC pressure detection)
go tool pprof -alloc_space http://localhost:6060/debug/pprof/heap

# Goroutine dump with full stack traces — pipe through grep
curl http://localhost:6060/debug/pprof/goroutine?debug=2 -o goroutines.txt
grep -A 20 "goroutine [0-9]* \[chan receive" goroutines.txt  # Find blocked goroutines

# Interactive web UI with flame graph (open browser at :8090)
go tool pprof -http=:8090 http://localhost:6060/debug/pprof/heap

export GODEBUG=gctrace=1
./my-service 2>&1 | grep "^gc"

# Sample output:
# gc 1 @0.005s 3%: 0.012+1.5+0.045 ms clock, 0.096+1.5/1.2/0+0.36 ms cpu, 4->4->2 MB, 5 MB goal, 8 P

package observability

import (
    "fmt"
    "runtime"
    "time"
)

// GoroutineLeakDetector alerts when goroutine count grows beyond a threshold
type GoroutineLeakDetector struct {
    baseline  int
    threshold int
}

func NewGoroutineLeakDetector(threshold int) *GoroutineLeakDetector {
    return &GoroutineLeakDetector{
        baseline:  runtime.NumGoroutine(),
        threshold: threshold,
    }
}

func (g *GoroutineLeakDetector) Check() bool {
    current := runtime.NumGoroutine()
    if current > g.baseline+g.threshold {
        fmt.Printf("ALERT: goroutines=%d (baseline=%d, threshold=+%d) — potential leak!\n",
            current, g.baseline, g.threshold)
        return true
    }
    return false
}

// RuntimeMetricsExporter periodically logs Go runtime metrics
type RuntimeMetricsExporter struct {
    interval time.Duration
}

func (e *RuntimeMetricsExporter) Start() {
    go func() {
        ticker := time.NewTicker(e.interval)
        defer ticker.Stop()
        for range ticker.C {
            var ms runtime.MemStats
            runtime.ReadMemStats(&ms)
            fmt.Printf("[runtime] goroutines=%d heap_inuse=%dMiB heap_alloc=%dMiB gc_total_pause=%dms gc_cycles=%d\n",
                runtime.NumGoroutine(),
                ms.HeapInuse/1024/1024,
                ms.HeapAlloc/1024/1024,
                ms.PauseTotalNs/1_000_000,
                ms.NumGC,
            )
        }
    }()
}

// ❌ LEAK: goroutine blocked on channel that nobody reads
func leakyHandler(w http.ResponseWriter, r *http.Request) {
    ch := make(chan Result) // Unbuffered channel
    go func() {
        result := expensiveWork()
        ch <- result // Blocks forever if caller returns early!
    }()
    
    select {
    case res := <-ch:
        w.Write(res.data)
    case <-time.After(1 * time.Second):
        http.Error(w, "timeout", 504) // Returns! goroutine still blocked on ch<-
    }
}

// ✅ FIXED: use context cancellation to unblock the goroutine
func fixedHandler(w http.ResponseWriter, r *http.Request) {
    ctx, cancel := context.WithTimeout(r.Context(), time.Second)
    defer cancel() // Always cancel — even on success
    
    ch := make(chan Result, 1) // Buffered — goroutine can always send
    go func() {
        select {
        case ch <- expensiveWork():
        case <-ctx.Done(): // Exit if context cancelled
        }
    }()
    
    select {
    case res := <-ch:
        w.Write(res.data)
    case <-ctx.Done():
        http.Error(w, "timeout", 504)
    }
}

package observability

import (
    "fmt"
    "net/http"
    _ "net/http/pprof" // Registers /debug/pprof/* handlers
    "time"
)

// StartObservabilityStack initializes all observability components
func StartObservabilityStack(pprofPort int, leakThreshold int) {
    // 1. pprof server (internal only)
    go func() {
        addr := fmt.Sprintf("localhost:%d", pprofPort)
        fmt.Printf("[observability] pprof server at http://%s/debug/pprof/\n", addr)
        http.ListenAndServe(addr, nil)
    }()

    // 2. Runtime metrics exporter (every 15s)
    exporter := &RuntimeMetricsExporter{interval: 15 * time.Second}
    exporter.Start()

    // 3. Goroutine leak detector (check every 30s)
    detector := NewGoroutineLeakDetector(leakThreshold)
    go func() {
        for range time.Tick(30 * time.Second) {
            detector.Check()
        }
    }()
}

# CPU (5–10% overhead during capture):
go tool pprof http://localhost:6060/debug/pprof/profile?seconds=30

# Heap (< 1% overhead):
go tool pprof -inuse_space http://localhost:6060/debug/pprof/heap

# Flame graph web UI:
go tool pprof -http=:8090 http://localhost:6060/debug/pprof/heap

# Interactive commands: top / list <func> / web / svg