fix: recover from Horizon bad_seq via account re-fetch, with chaos test coverage (closes #708)

src/services/stellarSubmissionService.chaos.test.ts

@@ -0,0 +1,106 @@
+import * as StellarSdk from '@stellar/stellar-sdk';
+
+jest.mock('../config/env', () => ({
+  env: {
+    STELLAR_HORIZON_URL: 'https://horizon-testnet.stellar.org',
+    STELLAR_NETWORK: 'testnet',
+    STELLAR_NETWORK_PASSPHRASE: undefined,
+    STELLAR_SERVER_SECRET: StellarSdk.Keypair.random().secret(),
+    STELLAR_MAX_FEE: '100',
+  },
+}));
+
+import { StellarSubmissionService } from './stellarSubmissionService';
+
+/**
+ * Mimics the shape of a Horizon "tx_bad_seq" error. If classifyStellarRPCFailure
+ * in stellarRpcFailure.ts detects BAD_SEQUENCE via a different field, adjust this
+ * shape to match — check that function first if these tests don't trigger BAD_SEQUENCE.
+ */
+function makeBadSeqError() {
+  const err: any = new Error('Bad Request');
+  err.response = {
+    status: 400,
+    data: {
+      status: 400,
+      type: 'transaction_failed',
+      extras: { result_codes: { transaction: 'tx_bad_seq' } },
+    },
+  };
+  return err;
+}
+
+function makeAccountNotFoundError() {
+  const err: any = new Error('Not Found');
+  err.response = { status: 404, data: { status: 404, type: 'not_found' } };
+  return err;
+}
+
+describe('StellarSubmissionService — bad_seq chaos', () => {
+  let service: StellarSubmissionService;
+  let getAccountMock: jest.Mock;
+  let sendTransactionMock: jest.Mock;
+
+  beforeEach(() => {
+    service = new StellarSubmissionService();
+
+    getAccountMock = jest.fn();
+    sendTransactionMock = jest.fn();
+    (service as any).server.getAccount = getAccountMock;
+    (service as any).server.sendTransaction = sendTransactionMock;
+
+    // Skip real backoff delays so tests run fast.
+    jest.spyOn(service as any, 'delay').mockResolvedValue(undefined);
+  });
+
+  const publicKey = (service: StellarSubmissionService) => service.getPublicKey();
+
+  it('recovers from an intermittent bad_seq by re-fetching the account and retrying', async () => {
+    getAccountMock
+      .mockResolvedValueOnce(new StellarSdk.Account(publicKey(service), '1')) // initial fetch
+      .mockResolvedValueOnce(new StellarSdk.Account(publicKey(service), '2')); // rebuild re-fetch
+
+    sendTransactionMock
+      .mockRejectedValueOnce(makeBadSeqError())
+      .mockResolvedValueOnce({ status: 'PENDING', hash: 'abc123' });
+
+    const result = await service.submitPayment(
+      StellarSdk.Keypair.random().publicKey(),
+      '10',
+    );
+
+    expect(result.status).toBe('PENDING');
+    expect(sendTransactionMock).toHaveBeenCalledTimes(2);
+    expect(getAccountMock).toHaveBeenCalledTimes(2);
+  });
+
+  it('bounds retries and fails loudly when bad_seq persists', async () => {
+    getAccountMock.mockResolvedValue(new StellarSdk.Account(publicKey(service), '1'));
+    sendTransactionMock.mockRejectedValue(makeBadSeqError());
+
+    await expect(
+      service.submitPayment(StellarSdk.Keypair.random().publicKey(), '10'),
+    ).rejects.toThrow();
+
+    // Bounded: retries must not exceed the configured budget (default maxRetries = 3).
+    const maxRetries = (service as any).maxRetries;
+    expect(sendTransactionMock.mock.calls.length).toBeLessThanOrEqual(maxRetries + 1);
+    expect(sendTransactionMock.mock.calls.length).toBeGreaterThan(1);
+  });
+
+  it('fails loudly, without hanging, on a stuck sequence gap (e.g. account merged away)', async () => {
+    getAccountMock
+      .mockResolvedValueOnce(new StellarSdk.Account(publicKey(service), '1')) // initial fetch
+      .mockRejectedValue(makeAccountNotFoundError()); // account no longer exists — merge scenario
+
+    sendTransactionMock.mockRejectedValue(makeBadSeqError());
+
+    await expect(
+      service.submitPayment(StellarSdk.Keypair.random().publicKey(), '10'),
+    ).rejects.toThrow();
+
+    // Must terminate (not hang) and must not retry unboundedly even though
+    // the recovery path itself is also failing.
+    expect(sendTransactionMock.mock.calls.length).toBeLessThan(20);
+  });
+});

src/services/stellarSubmissionService.ts

@@ -219,6 +219,32 @@ export class StellarSubmissionService {
 
         // Calculate exponential backoff delay
         const delayMs = this.calculateRetryDelay(failure.suggestedRetryDelayMs, attemptCount);
+        /**
+   * Rebuilds and re-signs a transaction against a freshly-fetched source account.
+   *
+   * A signed Stellar transaction has its sequence number baked into the signature,
+   * so resending the same signed transaction after a BAD_SEQUENCE failure will
+   * always fail identically. This re-fetches the current account state and
+   * rebuilds the transaction with the correct sequence before retrying.
+   */
+  private rebuildTransactionWithFreshSequence(
+    original: StellarSdk.Transaction,
+    freshAccount: StellarSdk.Account,
+  ): StellarSdk.Transaction {
+    const builder = new StellarSdk.TransactionBuilder(freshAccount, {
+      fee: original.fee,
+      networkPassphrase: original.networkPassphrase,
+      memo: original.memo,
+    });
+
+    for (const operation of original.operations) {
+      builder.addOperation(operation);
+    }
+
+    const rebuilt = builder.setTimeout(30).build();
+    rebuilt.sign(this.keypair);
+    return rebuilt;
+  }
         logger.debug('Retrying Stellar account retrieval', {
           publicKey,
           attemptCount,
@@ -245,8 +271,8 @@ export class StellarSubmissionService {
     transaction: StellarSdk.Transaction,
     context: StellarRPCFailureContext
   ): Promise<StellarSdk.rpc.Api.SendTransactionResponse> {
-    let attemptCount = context.attemptCount || 1;
-    const transactionHash = transaction.hash().toString('hex');
+  let attemptCount = context.attemptCount || 1;
+    let transactionHash = transaction.hash().toString('hex');
 
     while (attemptCount <= this.maxRetries) {
       try {
@@ -292,6 +318,25 @@ export class StellarSubmissionService {
         }
 
         this.logStellarFailure(failure);
+
+        // A bad-sequence failure means the signed transaction's sequence number is
+        // stale. Resending it unchanged will fail identically every time, so we
+        // must re-fetch the account and rebuild + re-sign before retrying.
+        if (failure.class === StellarRPCFailureClass.BAD_SEQUENCE) {
+          try {
+            const freshAccount = await this.getAccountWithRetry(this.keypair.publicKey(), {
+              ...context,
+              operation: 'get_account',
+              attemptCount,
+            });
+            transaction = this.rebuildTransactionWithFreshSequence(transaction, freshAccount);
+            transactionHash = transaction.hash().toString('hex');
+          } catch {
+            // Can't even re-fetch the account (e.g. it no longer exists) — surface
+            // the original bad-sequence failure rather than masking it.
+            throw this.createAppErrorFromFailure(failure);
+          }
+        }
 
         // Calculate exponential backoff delay
         const delayMs = this.calculateRetryDelay(failure.suggestedRetryDelayMs, attemptCount);