feat: update settlement workflow

application/settlement/worker.go

@@ -2,6 +2,7 @@ package settlement
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"time"
 
@@ -41,7 +42,13 @@ func (w Worker) RunOnce(ctx context.Context, limit int) error {
 		record.Attempts++
 		hash, submitErr := w.Submitter.Submit(ctx, record)
 		if submitErr != nil {
-			record = record.Retry(now, w.MaxAttempts, submitErr)
+			var permanent ledger.PermanentSubmissionError
+			if errors.As(submitErr, &permanent) {
+				record.Status = ledger.SettlementManualReview
+				record.LastError = submitErr.Error()
+			} else {
+				record = record.Retry(now, w.MaxAttempts, submitErr)
+			}
 		} else {
 			// Horizon returns success only after the transaction is accepted into
 			// a ledger, so a successful response is already confirmation.

domain/ledger/settlement.go

@@ -5,6 +5,11 @@ import (
 	"time"
 )
 
+type PermanentSubmissionError struct{ Err error }
+
+func (e PermanentSubmissionError) Error() string { return e.Err.Error() }
+func (e PermanentSubmissionError) Unwrap() error { return e.Err }
+
 type SettlementStatus string
 
 const (

infrastructure/kuknos/submitter.go

@@ -119,6 +119,9 @@ func (s Submitter) Submit(ctx context.Context, record ledger.Settlement) (string
 		err := fmt.Errorf("kuknos rejected transaction: status=%d code=%s", resp.StatusCode, response.Extras.ResultCodes.Transaction)
 		span.SetStatus(codes.Error, err.Error())
 		span.RecordError(err)
+		if response.Extras.ResultCodes.Transaction == "tx_bad_seq" {
+			return "", ledger.PermanentSubmissionError{Err: err}
+		}
 		return "", err
 	}
 	return response.Hash, nil