Part 2: Environment Setup with Docker, OSM & Golang

sudo apt-get update && sudo apt-get install -y osmium-tool curl

curl -O https://download.geofabrik.de/asia/vietnam-latest.osm.pbf

# Geographic Bounding Box: Lon 106.50 -> 106.90, Lat 10.60 -> 10.90
osmium extract -b 106.50,10.60,106.90,10.90 vietnam-latest.osm.pbf -o hcmc.osm.pbf

# Verify artifact size
ls -lh hcmc.osm.pbf
# Output: Compressed file reduced from 385MB to ~42MB!

routing-dev-cluster/
├── docker-compose.yml
├── data/
│   └── hcmc.osm.pbf
├── config/
│   └── graphhopper-config.yml
└── srtm/

graphhopper:
  datareader.file: "/data/hcmc.osm.pbf"
  graph.location: "/data/graph-cache"
  graph.encoded_values: "car_access, car_average_speed, motorcycle_access, motorcycle_average_speed, road_class, surface, toll"

  # Vehicle profile definitions
  profiles:
    - name: car
      custom_model_files: [car_custom.json]
    - name: motorcycle
      custom_model_files: [motorcycle_custom.json]

  profiles_ch:
    - profile: car
    - profile: motorcycle

  # Digital Elevation Model (SRTM 30m grid)
  graph.elevation.provider: srtm
  graph.elevation.cache_dir: "/data/srtm"
  graph.elevation.dataaccess: MMAP

server:
  application_connectors:
    - type: http
      port: 8989
      bind_host: 0.0.0.0

version: '3.8'

services:
  # GraphHopper Routing Service (Java 21 LTS)
  graphhopper:
    image: graphhopper/graphhopper:11.0
    container_name: routing-graphhopper
    restart: unless-stopped
    ports:
      - "8989:8989"
    volumes:
      - ./data:/data
      - ./config:/config
      - ./srtm:/data/srtm
    environment:
      # Allocate 4GB heap and enforce Java 21 Generational ZGC for low-latency garbage collection
      - JAVA_OPTS=-Xms4g -Xmx4g -XX:+UseZGC -XX:+ZGenerational
    command:
      - "--input"
      - "/data/hcmc.osm.pbf"
      - "--graph-location"
      - "/data/graph-cache"
      - "--config"
      - "/config/graphhopper-config.yml"
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:8989/health"]
      interval: 10s
      timeout: 5s
      retries: 5
      start_period: 40s

  # OSRM Routing Engine (C++ Contraction Hierarchies)
  osrm:
    image: osrm/osrm-backend:v5.27.1
    container_name: routing-osrm
    restart: unless-stopped
    ports:
      - "5000:5000"
    volumes:
      - ./data:/data
    # Critical: Enforce 2GB minimum shared memory segment size
    shm_size: 2gb
    command: osrm-routed --algorithm mld /data/hcmc.osrm
    depends_on:
      - graphhopper

  # In-Memory Spatial Semantic Cache
  redis:
    image: redis:7.4-alpine
    container_name: routing-redis
    restart: unless-stopped
    ports:
      - "6379:6379"
    command: redis-server --maxmemory 1gb --maxmemory-policy allkeys-lru --save ""

docker compose up -d
docker compose logs -f graphhopper

// Package main provides a resilient, multi-engine routing client in Go 1.25.
package main

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"iter"
	"log/slog"
	"math/rand/v2"
	"net/http"
	"os"
	"runtime"
	"time"
)

// GeoLocation models a discrete WGS-84 geographic coordinate pair.
type GeoLocation struct {
	Latitude  float64 `json:"lat"`
	Longitude float64 `json:"lon"`
}

// RouteResponse encapsulates calculated road path metrics.
type RouteResponse struct {
	DistanceMeters float64       `json:"distance_meters"`
	TimeDuration   time.Duration `json:"duration"`
	EngineType     string        `json:"engine_type"`
	StatusCode     int           `json:"status_code"`
}

// ClientOptions defines network connection thresholds and backoff limits.
type ClientOptions struct {
	GraphHopperBaseURL string
	OSRMBasedURL       string
	MaxRetries         int
	InitialBackoff     time.Duration
	RequestTimeout     time.Duration
}

// ResilientRoutingClient coordinates engine queries with automated failover and backoff.
type ResilientRoutingClient struct {
	opts       ClientOptions
	httpClient *http.Client
	logger     *slog.Logger
}

// NewResilientRoutingClient constructs a client with automated transport lifecycle cleanup.
func NewResilientRoutingClient(opts ClientOptions, logger *slog.Logger) (*ResilientRoutingClient, error) {
	if opts.MaxRetries <= 0 {
		opts.MaxRetries = 3
	}
	if opts.InitialBackoff <= 0 {
		opts.InitialBackoff = 50 * time.Millisecond
	}
	if opts.RequestTimeout <= 0 {
		opts.RequestTimeout = 2 * time.Second
	}

	transport := &http.Transport{
		MaxIdleConns:        500,
		MaxIdleConnsPerHost: 100,
		IdleConnTimeout:     90 * time.Second,
		DisableCompression: false,
	}

	client := &ResilientRoutingClient{
		opts: opts,
		httpClient: &http.Client{
			Transport: transport,
			Timeout:   opts.RequestTimeout,
		},
		logger: logger,
	}

	// Register deterministic transport resource cleanup via Go 1.25 runtime.AddCleanup
	runtime.AddCleanup(client, func(t *http.Transport) {
		t.CloseIdleConnections()
	}, transport)

	return client, nil
}

// CoordinatePairIterator generates coordinate pairs using Go 1.25 range-over-func.
func CoordinatePairIterator(pairs [][2]GeoLocation) iter.Seq2[int, [2]GeoLocation] {
	return func(yield func(int, [2]GeoLocation) bool) {
		for idx, pair := range pairs {
			if !yield(idx, pair) {
				return
			}
		}
	}
}

// Route executes path calculations with automated failover from GraphHopper to OSRM.
func (c *ResilientRoutingClient) Route(ctx context.Context, from, to GeoLocation) (*RouteResponse, error) {
	start := time.Now()

	// Primary Attempt: Query GraphHopper cluster
	resp, err := c.executeWithRetry(ctx, func(reqCtx context.Context) (*RouteResponse, error) {
		return c.callGraphHopper(reqCtx, from, to)
	})

	if err == nil {
		c.logger.Debug("GraphHopper query resolved successfully",
			slog.Group("metrics",
				slog.Float64("distance_m", resp.DistanceMeters),
				slog.Duration("latency", time.Since(start)),
			),
		)
		return resp, nil
	}

	c.logger.Warn("GraphHopper endpoint unavailable, failing over to OSRM",
		slog.String("primary_error", err.Error()),
	)

	// Secondary Fallback: Query OSRM cluster
	respOSRM, errOSRM := c.executeWithRetry(ctx, func(reqCtx context.Context) (*RouteResponse, error) {
		return c.callOSRM(reqCtx, from, to)
	})

	if errOSRM == nil {
		c.logger.Info("OSRM failover successful",
			slog.Group("metrics",
				slog.Float64("distance_m", respOSRM.DistanceMeters),
				slog.Duration("latency", time.Since(start)),
			),
		)
		return respOSRM, nil
	}

	return nil, fmt.Errorf("all routing backends exhausted: gh_err=%v, osrm_err=%w", err, errOSRM)
}

// executeWithRetry wraps execution with exponential backoff and randomized full jitter.
func (c *ResilientRoutingClient) executeWithRetry(
	ctx context.Context,
	fn func(context.Context) (*RouteResponse, error),
) (*RouteResponse, error) {
	var lastErr error
	backoff := c.opts.InitialBackoff

	for attempt := 0; attempt <= c.opts.MaxRetries; attempt++ {
		if attempt > 0 {
			jitter := time.Duration(rand.Int64N(int64(backoff)))
			sleepDuration := backoff + jitter

			select {
			case <-ctx.Done():
				return nil, ctx.Err()
			case <-time.After(sleepDuration):
			}
			backoff *= 2
		}

		res, err := fn(ctx)
		if err == nil {
			return res, nil
		}
		lastErr = err
	}

	return nil, lastErr
}

// callGraphHopper dispatches HTTP request to GraphHopper service.
func (c *ResilientRoutingClient) callGraphHopper(ctx context.Context, from, to GeoLocation) (*RouteResponse, error) {
	url := fmt.Sprintf("%s/route?point=%.6f,%.6f&point=%.6f,%.6f&profile=car&calc_points=false",
		c.opts.GraphHopperBaseURL, from.Latitude, from.Longitude, to.Latitude, to.Longitude)

	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
	if err != nil {
		return nil, err
	}

	resp, err := c.httpClient.Do(req)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	if resp.StatusCode != http.StatusOK {
		body, _ := io.ReadAll(resp.Body)
		return nil, fmt.Errorf("graphhopper returned status %d: %s", resp.StatusCode, string(body))
	}

	var ghResult struct {
		Paths []struct {
			Distance float64 `json:"distance"`
			Time     int64   `json:"time"`
		} `json:"paths"`
	}

	if err := json.NewDecoder(resp.Body).Decode(&ghResult); err != nil {
		return nil, err
	}

	if len(ghResult.Paths) == 0 {
		return nil, errors.New("no connecting route found in graphhopper")
	}

	return &RouteResponse{
		DistanceMeters: ghResult.Paths[0].Distance,
		TimeDuration:   time.Duration(ghResult.Paths[0].Time) * time.Millisecond,
		EngineType:     "GraphHopper-11.0",
		StatusCode:     resp.StatusCode,
	}, nil
}

// callOSRM dispatches HTTP request to OSRM service.
func (c *ResilientRoutingClient) callOSRM(ctx context.Context, from, to GeoLocation) (*RouteResponse, error) {
	url := fmt.Sprintf("%s/route/v1/driving/%.6f,%.6f;%.6f,%.6f?overview=false",
		c.opts.OSRMBasedURL, from.Longitude, from.Latitude, to.Longitude, to.Latitude)

	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
	if err != nil {
		return nil, err
	}

	resp, err := c.httpClient.Do(req)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	if resp.StatusCode != http.StatusOK {
		return nil, fmt.Errorf("osrm returned status %d", resp.StatusCode)
	}

	var osrmResult struct {
		Routes []struct {
			Distance float64 `json:"distance"`
			Duration float64 `json:"duration"`
		} `json:"routes"`
	}

	if err := json.NewDecoder(resp.Body).Decode(&osrmResult); err != nil {
		return nil, err
	}

	if len(osrmResult.Routes) == 0 {
		return nil, errors.New("no connecting route found in osrm")
	}

	return &RouteResponse{
		DistanceMeters: osrmResult.Routes[0].Distance,
		TimeDuration:   time.Duration(osrmResult.Routes[0].Duration * float64(time.Second)),
		EngineType:     "OSRM-5.27",
		StatusCode:     resp.StatusCode,
	}, nil
}

func main() {
	handler := slog.NewTextHandler(os.Stdout, &slog.HandlerOptions{Level: slog.LevelInfo})
	logger := slog.New(handler)

	opts := ClientOptions{
		GraphHopperBaseURL: "http://localhost:8989",
		OSRMBasedURL:       "http://localhost:5000",
		MaxRetries:         2,
		InitialBackoff:     100 * time.Millisecond,
		RequestTimeout:     1 * time.Second,
	}

	client, err := NewResilientRoutingClient(opts, logger)
	if err != nil {
		logger.Error("Client initialization failed", slog.String("error", err.Error()))
		os.Exit(1)
	}

	// Demonstration test queries in Ho Chi Minh City
	testPairs := [][2]GeoLocation{
		{
			{Latitude: 10.7769, Longitude: 106.7009}, // Ben Thanh Market
			{Latitude: 10.7798, Longitude: 106.6990}, // Independence Palace
		},
		{
			{Latitude: 10.7769, Longitude: 106.7009},
			{Latitude: 10.8231, Longitude: 106.6297}, // Tan Son Nhat Airport
		},
	}

	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
	defer cancel()

	for idx, pair := range CoordinatePairIterator(testPairs) {
		logger.Info("Executing route pair evaluation", slog.Int("index", idx))
		res, err := client.Route(ctx, pair[0], pair[1])
		if err != nil {
			logger.Error("Route evaluation error", slog.Int("index", idx), slog.String("error", err.Error()))
			continue
		}
		logger.Info("Route resolved successfully",
			slog.Int("index", idx),
			slog.String("engine", res.EngineType),
			slog.Float64("distance_meters", res.DistanceMeters),
			slog.Duration("estimated_duration", res.TimeDuration),
		)
	}
}

> 🔥 **[Production Failure]: 25-Minute Cold-Start Delay on GraphHopper Pod Evacuation**
> **Incident Window:** 09:15 - 09:45 UTC+7, August 22, 2025.
> **Impact Surface:** Entire logistics dispatch cluster in Singapore cloud region; 40% of distance matrix requests returned HTTP 503 Service Unavailable during the 25-minute degradation.
> **Symptom:** When the Kubernetes Cluster Autoscaler evicted 16 GraphHopper pods to rebalance worker nodes, replacement pods repeatedly entered `CrashLoopBackOff`; liveness probes timed out after 30 seconds.
> 
> **Root Cause Analysis (RCA):**
> 1. Infrastructure engineers configured the graph cache volume `/data/graph-cache` on a shared AWS EFS (Elastic File System) volume over NFS to avoid duplicating storage.
> 2. When 16 pods initialized simultaneously, each container opened and sequentially read thousands of small binary graph segments over the NFS protocol.
> 3. The shared EFS volume depleted its IOPS burst credit balance, causing network throughput to collapse to 1.2 MB/s.
> 4. JVM heap graph loading time increased from 20 seconds to 25 minutes, exceeding the Kubernetes `initialDelaySeconds: 60` threshold. The Kubelet dispatched `SIGKILL` signals, restarting pods repeatedly in an endless crash loop.
> 
> 📊 **Financial & Operational Impact:** 25 minutes of degraded dispatching; \$42,000 USD in delayed order fulfillment and driver cancellation penalties.
> 
> 📈 **Remediation & Prevention Architecture:**
> 1. **Immediate Triage:** Increased the Liveness Probe timeout threshold to 1,800 seconds (30 minutes) to allow active pods to complete graph loading over EFS.
> 2. **Prohibition of Shared Network Volumes for Graph Caches:** Migrated all graph cache volumes to **Local NVMe HostPath** volumes. Local NVMe read throughput exceeded 3,500 MB/s, reducing graph boot times to under 4 seconds.
> 3. **Init-Container Pre-Fetch Strategy:** Configured lightweight init-containers that pull graph archives from object storage (MinIO/S3) directly onto node-local NVMe drives before the primary routing container starts.