Add OpenTelemetry tracing and log correlation

infrastructure/config/config.go

@@ -17,12 +17,27 @@ type Config struct {
 	GRPC        GRPCConfig       `koanf:"grpc"`
 	Database    DatabaseConfig   `koanf:"db"`
 	Settlement  SettlementConfig `koanf:"settlement"`
+	Telemetry   TelemetryConfig  `koanf:"telemetry"`
 }
 
 type DashboardConfig struct {
-	Environment string         `koanf:"environment"`
-	HTTP        HTTPConfig     `koanf:"http"`
-	Database    DatabaseConfig `koanf:"db"`
+	Environment string          `koanf:"environment"`
+	HTTP        HTTPConfig      `koanf:"http"`
+	Database    DatabaseConfig  `koanf:"db"`
+	Telemetry   TelemetryConfig `koanf:"telemetry"`
+}
+
+type TelemetryConfig struct {
+	Enabled         bool              `koanf:"enabled"`
+	Protocol        string            `koanf:"protocol"`
+	Endpoint        string            `koanf:"endpoint"`
+	Insecure        bool              `koanf:"insecure"`
+	Headers         map[string]string `koanf:"headers"`
+	Sampler         string            `koanf:"sampler"`
+	SamplerArg      string            `koanf:"sampler-arg"`
+	ExportTimeout   time.Duration     `koanf:"export-timeout"`
+	BatchTimeout    time.Duration     `koanf:"batch-timeout"`
+	ShutdownTimeout time.Duration     `koanf:"shutdown-timeout"`
 }
 
 type GRPCConfig struct {

infrastructure/observability/logger.go

@@ -1,8 +1,11 @@
 package observability
 
 import (
+	"context"
 	"log/slog"
 	"os"
+
+	"go.opentelemetry.io/otel/trace"
 )
 
 // Configure installs the process-wide JSON logger used by every GL adapter.
@@ -24,7 +27,24 @@ func Configure(service string, level slog.Leveler) *slog.Logger {
 			return attr
 		},
 	})
-	logger := slog.New(handler).With("service", service)
+	logger := slog.New(&traceHandler{Handler: handler}).With("service", service)
 	slog.SetDefault(logger)
 	return logger
 }
+
+type traceHandler struct{ slog.Handler }
+
+func (h *traceHandler) Handle(ctx context.Context, record slog.Record) error {
+	spanContext := trace.SpanContextFromContext(ctx)
+	if spanContext.IsValid() {
+		record.AddAttrs(slog.String("trace_id", spanContext.TraceID().String()), slog.String("span_id", spanContext.SpanID().String()))
+	}
+	return h.Handler.Handle(ctx, record)
+}
+
+func (h *traceHandler) WithAttrs(attrs []slog.Attr) slog.Handler {
+	return &traceHandler{Handler: h.Handler.WithAttrs(attrs)}
+}
+func (h *traceHandler) WithGroup(name string) slog.Handler {
+	return &traceHandler{Handler: h.Handler.WithGroup(name)}
+}

infrastructure/observability/telemetry.go

@@ -0,0 +1,39 @@
+package observability
+
+import (
+	"context"
+	"fmt"
+	"log/slog"
+	"os"
+	"strconv"
+	"strings"
+	"time"
+
+	"gl/infrastructure/config"
+	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc"
+	"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp"
+	"go.opentelemetry.io/otel/propagation"
+	"go.opentelemetry.io/otel/sdk/resource"
+	sdktrace "go.opentelemetry.io/otel/sdk/trace"
+)
+
+// Setup configures tracing without making telemetry availability a startup dependency.
+func Setup(ctx context.Context, cfg config.TelemetryConfig, service, environment string) func(context.Context) {
+	if value := os.Getenv("OTEL_SERVICE_NAME"); value != "" { service = value }
+	if value := os.Getenv("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT"); value != "" { cfg.Endpoint = value } else if value := os.Getenv("OTEL_EXPORTER_OTLP_ENDPOINT"); value != "" { cfg.Endpoint = value }
+	if value := os.Getenv("OTEL_EXPORTER_OTLP_PROTOCOL"); value != "" { cfg.Protocol = value }
+	if value := os.Getenv("OTEL_TRACES_SAMPLER"); value != "" { cfg.Sampler = value }; if value := os.Getenv("OTEL_TRACES_SAMPLER_ARG"); value != "" { cfg.SamplerArg = value }
+	otel.SetTextMapPropagator(propagation.NewCompositeTextMapPropagator(propagation.TraceContext{}, propagation.Baggage{}))
+	if !cfg.Enabled || cfg.Endpoint == "" || (cfg.Protocol != "grpc" && cfg.Protocol != "http/protobuf") { if cfg.Enabled { slog.Warn("OpenTelemetry disabled due to invalid configuration") }; return func(context.Context) {} }
+	var exp sdktrace.SpanExporter; var err error
+	if cfg.Protocol == "grpc" { options := []otlptracegrpc.Option{otlptracegrpc.WithEndpoint(cfg.Endpoint), otlptracegrpc.WithTimeout(timeout(cfg.ExportTimeout)), otlptracegrpc.WithHeaders(cfg.Headers)}; if cfg.Insecure { options = append(options, otlptracegrpc.WithInsecure()) }; exp, err = otlptracegrpc.New(ctx, options...) } else { options := []otlptracehttp.Option{otlptracehttp.WithEndpoint(cfg.Endpoint), otlptracehttp.WithTimeout(timeout(cfg.ExportTimeout)), otlptracehttp.WithHeaders(cfg.Headers)}; if cfg.Insecure { options = append(options, otlptracehttp.WithInsecure()) }; exp, err = otlptracehttp.New(ctx, options...) }
+	if err != nil { slog.Warn("OpenTelemetry exporter unavailable; tracing disabled", "error", err); return func(context.Context) {} }
+	host, _ := os.Hostname(); res, err := resource.New(ctx, resource.WithAttributes(attribute.String("service.name", service), attribute.String("service.namespace", "darano"), attribute.String("deployment.environment.name", environment), attribute.String("host.name", host))); if err != nil { slog.Warn("OpenTelemetry resource setup failed", "error", err); return func(context.Context) {} }
+	ratio, parseErr := strconv.ParseFloat(cfg.SamplerArg, 64); if cfg.Sampler == "parentbased_traceidratio" && (parseErr != nil || ratio < 0 || ratio > 1) { slog.Warn("OpenTelemetry disabled due to invalid sampler", "error", fmt.Errorf("invalid sampler-arg")); return func(context.Context) {} }
+	provider := sdktrace.NewTracerProvider(sdktrace.WithResource(res), sdktrace.WithSampler(sdktrace.ParentBased(sdktrace.TraceIDRatioBased(ratio))), sdktrace.WithBatcher(exp, sdktrace.WithMaxQueueSize(2048), sdktrace.WithBatchTimeout(timeout(cfg.BatchTimeout)), sdktrace.WithExportTimeout(timeout(cfg.ExportTimeout))))
+	otel.SetTracerProvider(provider); return func(stop context.Context) { stop, cancel := context.WithTimeout(stop, timeout(cfg.ShutdownTimeout)); defer cancel(); if err := provider.Shutdown(stop); err != nil { slog.Warn("OpenTelemetry shutdown failed", "error", err) } }
+}
+func timeout(value time.Duration) time.Duration { if value > 0 { return value }; return 5*time.Second }
+var _ = strings.TrimSpace