fix(abstract-substrate): blake2_256 prehash signablePayload over 256 bytes

Substrate signs the raw encoded ExtrinsicPayload when it is at most 256
bytes, but signs the blake2_256 hash of the payload when it exceeds 256
bytes. The signablePayload getter previously always returned the raw
bytes, so for large extrinsics (e.g. POLYX nominate with 6+ validators)
the BitGoJS user signed the raw payload while the HSM signed the hash,
causing TSS signature combination to fail with InvalidSignature.

Return blake2_256(raw) (32 bytes) when the payload is larger than 256
bytes, and the raw bytes otherwise, so all downstream consumers
(sdk-coin-polyx, sdk-coin-dot, recovery flows) get the correct
signableHex without coin-specific logic.

Generated with [Linear](https://linear.app/bitgo/issue/SI-911/fix-signablepayload-to-apply-substrate-blake2-256-prehash-when#agent-session-0bfb5d75)

Co-authored-by: linear-code[bot] <222613912+linear-code[bot]@users.noreply.github.com>

Author: abhijit0943

modules/abstract-substrate/src/lib/transaction.ts

@@ -10,6 +10,7 @@ import {
 import { BaseCoin as CoinConfig } from '@bitgo/statics';
 import Keyring, { decodeAddress } from '@polkadot/keyring';
 import { u8aToBuffer } from '@polkadot/util';
+import { blake2AsU8a } from '@polkadot/util-crypto';
 import { construct, decode } from '@substrate/txwrapper-polkadot';
 import { UnsignedTransaction } from '@substrate/txwrapper-core';
 import { TypeRegistry } from '@substrate/txwrapper-core/lib/types';
@@ -533,12 +534,22 @@ export class Transaction extends BaseTransaction {
     this._substrateTransaction = tx;
   }
 
-  /** @inheritdoc **/
+  /**
+   * @inheritdoc
+   *
+   * Returns the bytes that are actually signed for this transaction. Substrate signs the raw
+   * encoded `ExtrinsicPayload` when it is at most 256 bytes, but for payloads larger than 256
+   * bytes it signs the blake2_256 hash of those bytes instead (see Polkadot.js
+   * `@polkadot/types/extrinsic/util` and the HSM firmware). Returning the raw payload for large
+   * extrinsics (e.g. nominate with many validators) would make the user and the HSM sign different
+   * messages, causing TSS signature combination to fail.
+   */
   get signablePayload(): Buffer {
     const extrinsicPayload = this._registry.createType('ExtrinsicPayload', this._substrateTransaction, {
       version: EXTRINSIC_VERSION,
     });
-    return u8aToBuffer(extrinsicPayload.toU8a({ method: true }));
+    const raw = extrinsicPayload.toU8a({ method: true });
+    return u8aToBuffer(raw.length > 256 ? blake2AsU8a(raw, 256) : raw);
   }
 
   /**

modules/sdk-coin-polyx/test/unit/transactionBuilder/nominateBuilder.ts

@@ -179,6 +179,38 @@ describe('Polyx Nominate Builder', function () {
     });
   });
 
+  describe('signablePayload (Substrate 256-byte blake2 rule)', () => {
+    const buildNominateTx = async (validators: string[]) => {
+      const material = utils.getMaterial(coins.get('tpolyx').network.type);
+      const nominateBuilder = factory.getNominateBuilder();
+      nominateBuilder
+        .validators(validators)
+        .sender({ address: senderAddress })
+        .validity({ firstValid: 3933, maxDuration: 64 })
+        .referenceBlock('0x149799bc9602cb5cf201f3425fb8d253b2d4e61fc119dcab3249f307f594754d')
+        .sequenceId({ name: 'Nonce', keyword: 'nonce', value: 15 })
+        .fee({ amount: 0, type: 'tip' })
+        .material(material);
+      return nominateBuilder.build();
+    };
+
+    it('should return raw payload bytes when the extrinsic is at most 256 bytes', async () => {
+      const tx = await buildNominateTx([validatorAddress, validatorAddress2]);
+      const signablePayload = tx.signablePayload;
+      // small nominate extrinsic stays under the 256-byte threshold, so it is signed as-is
+      signablePayload.length.should.be.belowOrEqual(256);
+      signablePayload.length.should.not.equal(32);
+    });
+
+    it('should return the 32-byte blake2_256 hash when the extrinsic exceeds 256 bytes', async () => {
+      // 6+ validators (~33 bytes each) pushes the nominate extrinsic over the 256-byte threshold
+      const manyValidators = Array(8).fill(validatorAddress);
+      const tx = await buildNominateTx(manyValidators);
+      const signablePayload = tx.signablePayload;
+      should.equal(signablePayload.length, 32);
+    });
+  });
+
   describe('factory routing', () => {
     it('should route raw nominate extrinsic to NominateBuilder', () => {
       const resolvedBuilder = factory.from(nominateTx.signed);