feat: add durable Kuknos settlement coordination

cmd/dashboard-loadtest/main_test.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"context"
+	"net"
 	"net/http"
 	"net/http/httptest"
 	"sync/atomic"
@@ -10,7 +11,7 @@ import (
 )
 
 func TestRunSendsConcurrentRequestsAcrossTargets(t *testing.T) {
-	server := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, _ *http.Request) {
+	server := newIPv4TestServer(http.HandlerFunc(func(response http.ResponseWriter, _ *http.Request) {
 		response.WriteHeader(http.StatusOK)
 		_, _ = response.Write([]byte("ok"))
 	}))
@@ -49,7 +50,7 @@ func TestPercentileUsesNearestRank(t *testing.T) {
 
 func TestRunDrainsInflightRequestAfterDuration(t *testing.T) {
 	var canceled atomic.Bool
-	server := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) {
+	server := newIPv4TestServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) {
 		time.Sleep(20 * time.Millisecond)
 		if request.Context().Err() != nil {
 			canceled.Store(true)
@@ -66,3 +67,14 @@ func TestRunDrainsInflightRequestAfterDuration(t *testing.T) {
 		t.Fatalf("in-flight request was not drained cleanly: %+v", result)
 	}
 }
+
+func newIPv4TestServer(handler http.Handler) *httptest.Server {
+	listener, err := net.Listen("tcp4", "127.0.0.1:0")
+	if err != nil {
+		panic(err)
+	}
+	server := httptest.NewUnstartedServer(handler)
+	server.Listener = listener
+	server.Start()
+	return server
+}

cmd/gl/main.go

@@ -7,10 +7,13 @@ import (
 	"os"
 	"os/signal"
 	"syscall"
+	"time"
 
 	"gl/application/health"
 	applicationledger "gl/application/ledger"
+	applicationsettlement "gl/application/settlement"
 	"gl/infrastructure/config"
+	"gl/infrastructure/kuknos"
 	"gl/infrastructure/observability"
 	"gl/infrastructure/postgres"
 	grpcadapter "gl/interface/grpc"
@@ -42,7 +45,21 @@ func main() {
 	}
 
 	repository := postgres.NewJournalRepository(database)
-	handler := grpcadapter.NewHandler(health.NewService(database), applicationledger.NewService(repository, nil))
+	settlementRepository := postgres.NewSettlementRepository(database)
+	settlementService := applicationsettlement.NewService(settlementRepository)
+	var healthService *health.Service
+	if cfg.Settlement.Enabled {
+		healthService = health.NewService(database, settlementRepository)
+	} else {
+		healthService = health.NewService(database)
+	}
+	handler := grpcadapter.NewHandler(healthService, applicationledger.NewService(repository, nil), settlementService).WithSettlementAdminToken(cfg.Settlement.AdminToken)
+	if cfg.Settlement.Enabled {
+		worker := applicationsettlement.Worker{Repository: settlementRepository, Submitter: kuknos.Submitter{Endpoint: cfg.Settlement.Endpoint}, WorkerID: cfg.Settlement.WorkerID, MaxAttempts: cfg.Settlement.MaxAttempts}
+		go runSettlementWorker(ctx, worker, cfg.Settlement.PollInterval, cfg.Settlement.BatchSize)
+		go runSettlementReconciliation(ctx, settlementRepository)
+		slog.Info("Kuknos settlement worker enabled", "endpoint", cfg.Settlement.Endpoint, "worker_id", cfg.Settlement.WorkerID)
+	}
 	slog.Info("starting GL gRPC service", "host", cfg.GRPC.Host, "port", cfg.GRPC.Port)
 	serverConfig := grpcadapter.ServerConfig{
 		Host:            cfg.GRPC.Host,
@@ -54,3 +71,48 @@ func main() {
 		os.Exit(1)
 	}
 }
+
+func runSettlementReconciliation(ctx context.Context, repository *postgres.SettlementRepository) {
+	run := func() {
+		stats, err := repository.ReconcileConfirmationEvidence(ctx, 10*time.Minute)
+		if err != nil {
+			slog.ErrorContext(ctx, "settlement reconciliation failed", "error", err)
+			return
+		}
+		if stats.Detected > 0 || stats.Resolved > 0 || stats.Open > 0 {
+			slog.WarnContext(ctx, "settlement reconciliation completed", "detected", stats.Detected, "resolved", stats.Resolved, "open", stats.Open)
+		}
+	}
+	run()
+	ticker := time.NewTicker(time.Minute)
+	defer ticker.Stop()
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-ticker.C:
+			run()
+		}
+	}
+}
+
+func runSettlementWorker(ctx context.Context, worker applicationsettlement.Worker, interval time.Duration, batchSize int) {
+	if interval <= 0 {
+		interval = 5 * time.Second
+	}
+	if batchSize <= 0 {
+		batchSize = 20
+	}
+	ticker := time.NewTicker(interval)
+	defer ticker.Stop()
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-ticker.C:
+			if err := worker.RunOnce(ctx, batchSize); err != nil {
+				slog.ErrorContext(ctx, "settlement worker cycle failed", "error", err)
+			}
+		}
+	}
+}

cmd/healthcheck/main.go

@@ -0,0 +1,30 @@
+package main
+
+import (
+	"context"
+	"flag"
+	"os"
+	"time"
+
+	"gl/infrastructure/config"
+	"gl/infrastructure/postgres"
+)
+
+func main() {
+	configPath := flag.String("conf", "/app/config.toml", "path to the GL TOML configuration file")
+	flag.Parse()
+	ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
+	defer cancel()
+	cfg, err := config.Load(*configPath)
+	if err != nil {
+		os.Exit(1)
+	}
+	database, err := postgres.Open(ctx, cfg.Database)
+	if err != nil {
+		os.Exit(1)
+	}
+	defer database.Close()
+	if err := database.Ping(ctx); err != nil {
+		os.Exit(1)
+	}
+}