fix(abstract-substrate): blake2_256 prehash signablePayload over 256 bytes

Substrate signs the raw encoded ExtrinsicPayload when it is at most 256
bytes, but signs the blake2_256 hash of the payload when it exceeds 256
bytes. The signablePayload getter previously always returned the raw
bytes, so for large extrinsics (e.g. POLYX nominate with 6+ validators)
the BitGoJS user signed the raw payload while the HSM signed the hash,
causing TSS signature combination to fail with InvalidSignature.

Return blake2_256(raw) (32 bytes) when the payload is larger than 256
bytes, and the raw bytes otherwise, so all downstream consumers
(sdk-coin-polyx, sdk-coin-dot, recovery flows) get the correct
signableHex without coin-specific logic.

Generated with [Linear](https://linear.app/bitgo/issue/SI-911/fix-signablepayload-to-apply-substrate-blake2-256-prehash-when#agent-session-0bfb5d75)

Co-authored-by: linear-code[bot] <222613912+linear-code[bot]@users.noreply.github.com>

Author: abhijit0943

modules/abstract-substrate/src/lib/constants.ts

@@ -1 +1,7 @@
 export const DEFAULT_SUBSTRATE_PREFIX = 42;
+
+/**
+ * Substrate signs the raw encoded `ExtrinsicPayload` only when it is at most this many bytes;
+ * larger payloads are signed as their blake2_256 hash instead. See `getSubstrateSigningBytes`.
+ */
+export const MAX_RAW_SIGNING_PAYLOAD_BYTES = 256;

modules/abstract-substrate/src/lib/transaction.ts

@@ -9,7 +9,6 @@ import {
 } from '@bitgo/sdk-core';
 import { BaseCoin as CoinConfig } from '@bitgo/statics';
 import Keyring, { decodeAddress } from '@polkadot/keyring';
-import { u8aToBuffer } from '@polkadot/util';
 import { construct, decode } from '@substrate/txwrapper-polkadot';
 import { UnsignedTransaction } from '@substrate/txwrapper-core';
 import { TypeRegistry } from '@substrate/txwrapper-core/lib/types';
@@ -533,12 +532,21 @@ export class Transaction extends BaseTransaction {
     this._substrateTransaction = tx;
   }
 
-  /** @inheritdoc **/
+  /**
+   * @inheritdoc
+   *
+   * Returns the bytes that are actually signed for this transaction. Substrate signs the raw
+   * encoded `ExtrinsicPayload` when it is at most 256 bytes, but for payloads larger than 256
+   * bytes it signs the blake2_256 hash of those bytes instead (see Polkadot.js
+   * `@polkadot/types/extrinsic/util` and the HSM firmware). Returning the raw payload for large
+   * extrinsics (e.g. nominate with many validators) would make the user and the HSM sign different
+   * messages, causing TSS signature combination to fail.
+   */
   get signablePayload(): Buffer {
     const extrinsicPayload = this._registry.createType('ExtrinsicPayload', this._substrateTransaction, {
       version: EXTRINSIC_VERSION,
     });
-    return u8aToBuffer(extrinsicPayload.toU8a({ method: true }));
+    return utils.getSubstrateSigningBytes(extrinsicPayload.toU8a({ method: true }));
   }
 
   /**

modules/abstract-substrate/src/lib/utils.ts

@@ -4,8 +4,8 @@ import { decodeAddress, encodeAddress, Keyring } from '@polkadot/keyring';
 import { decodePair } from '@polkadot/keyring/pair/decode';
 import { KeyringPair } from '@polkadot/keyring/types';
 import { EXTRINSIC_VERSION } from '@polkadot/types/extrinsic/v4/Extrinsic';
-import { hexToU8a, isHex, u8aToHex, u8aToU8a } from '@polkadot/util';
-import { base64Decode, signatureVerify } from '@polkadot/util-crypto';
+import { hexToU8a, isHex, u8aToBuffer, u8aToHex, u8aToU8a } from '@polkadot/util';
+import { base64Decode, blake2AsU8a, signatureVerify } from '@polkadot/util-crypto';
 import { UnsignedTransaction } from '@substrate/txwrapper-core';
 import { DecodedSignedTx, DecodedSigningPayload, TypeRegistry } from '@substrate/txwrapper-core/lib/types';
 import { construct, decode } from '@substrate/txwrapper-polkadot';
@@ -31,6 +31,7 @@ import {
   MoveStakeArgs,
 } from './iface';
 import { SingletonRegistry } from './singletonRegistry';
+import { MAX_RAW_SIGNING_PAYLOAD_BYTES } from './constants';
 
 export class Utils implements BaseUtils {
   /** @inheritdoc */
@@ -343,6 +344,22 @@ export class Utils implements BaseUtils {
       throw new Error(`Failed to decode transaction: ${error}`);
     }
   }
+
+  /**
+   * Returns the bytes that Substrate actually signs for a given raw encoded `ExtrinsicPayload`.
+   *
+   * Substrate signs the raw payload as-is when it is at most {@link MAX_RAW_SIGNING_PAYLOAD_BYTES}
+   * bytes, but for larger payloads it signs the 32-byte blake2_256 hash of those bytes instead
+   * (see Polkadot.js `@polkadot/types/extrinsic/util` and the HSM firmware). Using this helper
+   * ensures the user and the HSM sign the same message, which is required for TSS signature
+   * combination to succeed on large extrinsics (e.g. nominate with many validators).
+   *
+   * @param {Uint8Array} raw The raw encoded extrinsic payload bytes.
+   * @returns {Buffer} The bytes to sign: the raw payload, or its blake2_256 hash when oversized.
+   */
+  getSubstrateSigningBytes(raw: Uint8Array): Buffer {
+    return u8aToBuffer(raw.length > MAX_RAW_SIGNING_PAYLOAD_BYTES ? blake2AsU8a(raw, 256) : raw);
+  }
 }
 
 const utils = new Utils();

modules/sdk-coin-polyx/test/unit/transactionBuilder/batchStakingBuilder.ts

@@ -234,6 +234,38 @@ describe('Polyx Batch Builder', function () {
     });
   });
 
+  describe('signablePayload (Substrate 256-byte blake2 rule)', function () {
+    const buildBatchTx = async (numValidators: number) => {
+      const batchBuilder = factory.getBatchBuilder();
+      batchBuilder
+        .amount(testAmount)
+        .controller({ address: controllerAddress })
+        .payee('Staked')
+        .validators(Array(numValidators).fill(validatorAddress))
+        .sender({ address: senderAddress })
+        .validity({ firstValid: 3933, maxDuration: 64 })
+        .referenceBlock('0x149799bc9602cb5cf201f3425fb8d253b2d4e61fc119dcab3249f307f594754d')
+        .sequenceId({ name: 'Nonce', keyword: 'nonce', value: 100 });
+      batchBuilder.material(utils.getMaterial(coins.get('tpolyx').network.type));
+      return batchBuilder.build();
+    };
+
+    it('should return raw payload bytes when the batch extrinsic is at most 256 bytes', async () => {
+      // bond + nominate with a single validator stays under the 256-byte threshold
+      const tx = await buildBatchTx(1);
+      const signablePayload = tx.signablePayload;
+      signablePayload.length.should.be.belowOrEqual(256);
+      signablePayload.length.should.not.equal(32);
+    });
+
+    it('should return the 32-byte blake2_256 hash when the batch extrinsic exceeds 256 bytes', async () => {
+      // bond + nominate with many validators pushes the batch over the 256-byte threshold
+      const tx = await buildBatchTx(6);
+      const signablePayload = tx.signablePayload;
+      should.equal(signablePayload.length, 32);
+    });
+  });
+
   describe('From Raw Transaction', function () {
     it('should rebuild from real batch transaction', async () => {
       // First build a transaction to get a real raw transaction

modules/sdk-coin-polyx/test/unit/transactionBuilder/nominateBuilder.ts

@@ -179,6 +179,52 @@ describe('Polyx Nominate Builder', function () {
     });
   });
 
+  describe('signablePayload (Substrate 256-byte blake2 rule)', () => {
+    const buildNominateTx = async (validators: string[]) => {
+      const material = utils.getMaterial(coins.get('tpolyx').network.type);
+      const nominateBuilder = factory.getNominateBuilder();
+      nominateBuilder
+        .validators(validators)
+        .sender({ address: senderAddress })
+        .validity({ firstValid: 3933, maxDuration: 64 })
+        .referenceBlock('0x149799bc9602cb5cf201f3425fb8d253b2d4e61fc119dcab3249f307f594754d')
+        .sequenceId({ name: 'Nonce', keyword: 'nonce', value: 15 })
+        .fee({ amount: 0, type: 'tip' })
+        .material(material);
+      return nominateBuilder.build();
+    };
+
+    it('should return raw payload bytes when the extrinsic is at most 256 bytes', async () => {
+      const tx = await buildNominateTx([validatorAddress, validatorAddress2]);
+      const signablePayload = tx.signablePayload;
+      // small nominate extrinsic stays under the 256-byte threshold, so it is signed as-is
+      signablePayload.length.should.be.belowOrEqual(256);
+      signablePayload.length.should.not.equal(32);
+    });
+
+    it('should return the 32-byte blake2_256 hash when the extrinsic exceeds 256 bytes', async () => {
+      // 6+ validators (~33 bytes each) pushes the nominate extrinsic over the 256-byte threshold
+      const manyValidators = Array(8).fill(validatorAddress);
+      const tx = await buildNominateTx(manyValidators);
+      const signablePayload = tx.signablePayload;
+      should.equal(signablePayload.length, 32);
+    });
+
+    // The 256-byte boundary is impractical to hit with a real extrinsic, so exercise the
+    // raw-vs-hash decision directly through the shared Substrate signing-bytes helper.
+    it('should keep payloads of exactly 256 bytes raw and only hash strictly larger ones', () => {
+      const at = utils.getSubstrateSigningBytes(new Uint8Array(256).fill(7));
+      should.equal(at.length, 256);
+      at.should.deepEqual(Buffer.alloc(256, 7));
+
+      const below = utils.getSubstrateSigningBytes(new Uint8Array(255).fill(7));
+      should.equal(below.length, 255);
+
+      const above = utils.getSubstrateSigningBytes(new Uint8Array(257).fill(7));
+      should.equal(above.length, 32);
+    });
+  });
+
   describe('factory routing', () => {
     it('should route raw nominate extrinsic to NominateBuilder', () => {
       const resolvedBuilder = factory.from(nominateTx.signed);