From 1635750b9e65772aec118faa7ae8edbaf1f30890 Mon Sep 17 00:00:00 2001 From: Ravi Kiran Date: Fri, 6 Mar 2026 18:15:45 +0530 Subject: [PATCH 1/8] Fix cursor pagination with optional tie-breaker --- .../api/src/app/flows/flow/flow.service.ts | 5 + .../app/helper/pagination/build-paginator.ts | 13 + .../src/app/helper/pagination/paginator.ts | 263 ++++++++++++++++-- 3 files changed, 253 insertions(+), 28 deletions(-) diff --git a/packages/server/api/src/app/flows/flow/flow.service.ts b/packages/server/api/src/app/flows/flow/flow.service.ts index 87584b58d6..4be7134e32 100644 --- a/packages/server/api/src/app/flows/flow/flow.service.ts +++ b/packages/server/api/src/app/flows/flow/flow.service.ts @@ -145,6 +145,11 @@ export const flowService = { columnName: 'fv.updated', columnType: 'timestamp with time zone', }, + customPaginationTieBreakerColumn: { + columnPath: 'id', + columnName: 'flow.id', + columnType: 'string', + }, }); const queryWhere: Record = { diff --git a/packages/server/api/src/app/helper/pagination/build-paginator.ts b/packages/server/api/src/app/helper/pagination/build-paginator.ts index 5a7c364ac1..9d3486b342 100644 --- a/packages/server/api/src/app/helper/pagination/build-paginator.ts +++ b/packages/server/api/src/app/helper/pagination/build-paginator.ts @@ -17,6 +17,11 @@ export type PaginationOptions = { columnName: string; columnType?: string; }; + customPaginationTieBreakerColumn?: { + columnPath: string; + columnName: string; + columnType?: string; + }; }; export function buildPaginator( @@ -40,6 +45,14 @@ export function buildPaginator( ); } + if (options.customPaginationTieBreakerColumn) { + paginator.setPaginationTieBreakerColumn( + options.customPaginationTieBreakerColumn.columnPath, + options.customPaginationTieBreakerColumn.columnName, + options.customPaginationTieBreakerColumn.columnType, + ); + } + if (query.afterCursor) { paginator.setAfterCursor(query.afterCursor); } diff --git a/packages/server/api/src/app/helper/pagination/paginator.ts b/packages/server/api/src/app/helper/pagination/paginator.ts index 2f43fc285a..903a168460 100644 --- a/packages/server/api/src/app/helper/pagination/paginator.ts +++ b/packages/server/api/src/app/helper/pagination/paginator.ts @@ -31,8 +31,16 @@ export type PagingResult = { cursor: CursorResult; }; +type CursorContext = { + primaryColumnName: string; + primaryParamName: string; + tieBreakerColumnName: string | null; + tieBreakerParamName: string | null; +}; + const PAGINATION_KEY = 'created'; const CUSTOM_PAGINATION_KEY = 'custom_pagination'; +const CUSTOM_PAGINATION_TIE_BREAKER_KEY = 'custom_pagination_tie_breaker'; const DEFAULT_TIMESTAMP_TYPE = 'timestamp with time zone'; export default class Paginator { @@ -56,6 +64,12 @@ export default class Paginator { private paginationColumnType: string | null = null; + private paginationTieBreakerColumnPath: string | null = null; + + private paginationTieBreakerColumnName: string | null = null; + + private paginationTieBreakerColumnType: string | null = null; + public constructor(private readonly entity: EntitySchema) {} public setPaginationColumn( @@ -72,6 +86,16 @@ export default class Paginator { this.alias = alias; } + public setPaginationTieBreakerColumn( + columnPath: string, + columnName: string, + columnType = 'string', + ): void { + this.paginationTieBreakerColumnPath = columnPath; + this.paginationTieBreakerColumnName = columnName; + this.paginationTieBreakerColumnType = columnType; + } + public setAfterCursor(cursor: string): void { this.afterCursor = cursor; } @@ -146,10 +170,10 @@ export default class Paginator { const cursors: CursorParam = {}; const clonedBuilder = new SelectQueryBuilder(builder); - if (this.hasAfterCursor()) { - Object.assign(cursors, this.decode(this.afterCursor!)); - } else if (this.hasBeforeCursor()) { - Object.assign(cursors, this.decode(this.beforeCursor!)); + if (this.afterCursor !== null) { + Object.assign(cursors, this.decode(this.afterCursor)); + } else if (this.beforeCursor !== null) { + Object.assign(cursors, this.decode(this.beforeCursor)); } if (Object.keys(cursors).length > 0) { @@ -169,30 +193,28 @@ export default class Paginator { where: WhereExpressionBuilder, cursors: CursorParam, ): void { - const dbType = system.get(AppSystemProp.DB_TYPE); + const dbType = this.getSupportedDbType(); const operator = this.getOperator(); - let queryString: string; - - const isCustomColumn = - this.paginationColumnName && cursors[CUSTOM_PAGINATION_KEY]; - const columnName = isCustomColumn - ? this.paginationColumnName - : `${this.alias}.${PAGINATION_KEY}`; - const paramName = isCustomColumn ? CUSTOM_PAGINATION_KEY : PAGINATION_KEY; - - if (dbType === DatabaseType.SQLITE3) { - queryString = `${columnName} ${operator} :${paramName}`; - } else if (dbType === DatabaseType.POSTGRES) { - if (this.hasBeforeCursor() && !this.hasAfterCursor()) { - queryString = `${columnName} ${operator} (:${paramName}::timestamp + INTERVAL '1 millisecond')`; - } else { - queryString = `${columnName} ${operator} :${paramName}::timestamp`; - } - } else { - throw new Error('Unsupported database type'); + const context = this.resolveCursorContext(cursors); + + if (context.tieBreakerColumnName && context.tieBreakerParamName) { + this.applyCompositeCursorFilter( + where, + cursors, + dbType, + operator, + context, + ); + return; } - where.orWhere(queryString, cursors); + this.applySingleColumnCursorFilter( + where, + cursors, + dbType, + operator, + context, + ); } private getOperator(): string { @@ -222,6 +244,10 @@ export default class Paginator { orderByCondition[`${this.alias}.${PAGINATION_KEY}`] = order; } + if (this.paginationTieBreakerColumnName) { + orderByCondition[this.paginationTieBreakerColumnName] = order; + } + return orderByCondition; } @@ -247,7 +273,7 @@ export default class Paginator { } const value = getValueByPath(entity, this.paginationColumnPath); - if (!value) { + if (value === null || value === undefined) { throw new Error( `Pagination column not found at path: ${this.paginationColumnPath}`, ); @@ -257,9 +283,31 @@ export default class Paginator { this.paginationColumnType || DEFAULT_TIMESTAMP_TYPE, value, ); - const payload = `${CUSTOM_PAGINATION_KEY}:${encodedValue}`; + const payload = [`${CUSTOM_PAGINATION_KEY}:${encodedValue}`]; - return btoa(payload); + if ( + this.paginationTieBreakerColumnPath && + this.paginationTieBreakerColumnName + ) { + const tieBreakerValue = getValueByPath( + entity, + this.paginationTieBreakerColumnPath, + ); + if (tieBreakerValue === null || tieBreakerValue === undefined) { + throw new Error( + `Pagination tie breaker column not found at path: ${this.paginationTieBreakerColumnPath}`, + ); + } + const encodedTieBreakerValue = encodeByType( + this.paginationTieBreakerColumnType || 'string', + tieBreakerValue, + ); + payload.push( + `${CUSTOM_PAGINATION_TIE_BREAKER_KEY}:${encodedTieBreakerValue}`, + ); + } + + return btoa(payload.join(',')); } private decode(cursor: string): CursorParam { @@ -279,6 +327,9 @@ export default class Paginator { if (key === CUSTOM_PAGINATION_KEY) { return this.paginationColumnType || DEFAULT_TIMESTAMP_TYPE; } + if (key === CUSTOM_PAGINATION_TIE_BREAKER_KEY) { + return this.paginationTieBreakerColumnType || 'string'; + } const col = this.entity.options.columns[key]; if (col === undefined) { @@ -291,6 +342,162 @@ export default class Paginator { return order === Order.ASC ? Order.DESC : Order.ASC; } + private buildComparisonClause({ + dbType, + columnName, + paramName, + operator, + }: { + dbType: DatabaseType; + columnName: string; + paramName: string; + operator: string; + }): string { + if (dbType === DatabaseType.SQLITE3) { + return `${columnName} ${operator} :${paramName}`; + } + + if (dbType === DatabaseType.POSTGRES) { + const type = this.getEntityPropertyType(paramName); + if (this.isTimestampType(type)) { + if (operator === '<') { + return `${columnName} < :${paramName}::timestamptz`; + } + if (operator === '>') { + return `${columnName} >= (:${paramName}::timestamptz + INTERVAL '1 millisecond')`; + } + if (operator === '=') { + return `(${columnName} >= :${paramName}::timestamptz AND ${columnName} < (:${paramName}::timestamptz + INTERVAL '1 millisecond'))`; + } + return `${columnName} ${operator} :${paramName}::timestamptz`; + } + return `${columnName} ${operator} :${paramName}`; + } + + throw new Error('Unsupported database type'); + } + + private isTimestampType(type: string): boolean { + return ( + type === 'timestamp with time zone' || + type === 'datetime' || + type === 'date' + ); + } + + private getSupportedDbType(): DatabaseType { + const dbType = system.get(AppSystemProp.DB_TYPE); + if (dbType === DatabaseType.SQLITE3 || dbType === DatabaseType.POSTGRES) { + return dbType; + } + throw new Error('Unsupported database type'); + } + + private resolveCursorContext(cursors: CursorParam): CursorContext { + const customPaginationColumnName = this.paginationColumnName; + const hasCustomPaginationCursor = + customPaginationColumnName !== null && + cursors[CUSTOM_PAGINATION_KEY] !== undefined; + + const primaryColumnName = + hasCustomPaginationCursor && customPaginationColumnName + ? customPaginationColumnName + : `${this.alias}.${PAGINATION_KEY}`; + const primaryParamName = hasCustomPaginationCursor + ? CUSTOM_PAGINATION_KEY + : PAGINATION_KEY; + + const hasCustomTieBreakerCursor = + this.paginationTieBreakerColumnName !== null && + cursors[CUSTOM_PAGINATION_TIE_BREAKER_KEY] !== undefined; + + if (hasCustomPaginationCursor && hasCustomTieBreakerCursor) { + return { + primaryColumnName, + primaryParamName, + tieBreakerColumnName: this.paginationTieBreakerColumnName, + tieBreakerParamName: CUSTOM_PAGINATION_TIE_BREAKER_KEY, + }; + } + + return { + primaryColumnName, + primaryParamName, + tieBreakerColumnName: null, + tieBreakerParamName: null, + }; + } + + private applySingleColumnCursorFilter( + where: WhereExpressionBuilder, + cursors: CursorParam, + dbType: DatabaseType, + operator: string, + context: CursorContext, + ): void { + where.orWhere( + this.buildComparisonClause({ + dbType, + columnName: context.primaryColumnName, + paramName: context.primaryParamName, + operator, + }), + cursors, + ); + } + + private applyCompositeCursorFilter( + where: WhereExpressionBuilder, + cursors: CursorParam, + dbType: DatabaseType, + operator: string, + context: CursorContext, + ): void { + const { + primaryColumnName, + primaryParamName, + tieBreakerColumnName, + tieBreakerParamName, + } = context; + if (!tieBreakerColumnName || !tieBreakerParamName) { + throw new Error('Pagination tie breaker context is not configured'); + } + + where.orWhere( + this.buildComparisonClause({ + dbType, + columnName: primaryColumnName, + paramName: primaryParamName, + operator, + }), + cursors, + ); + + // Lexicographic cursor compare: primary equals, then compare tie-breaker. + where.orWhere( + new Brackets((nestedWhere) => { + nestedWhere.where( + this.buildComparisonClause({ + dbType, + columnName: primaryColumnName, + paramName: primaryParamName, + operator: '=', + }), + cursors, + ); + nestedWhere.andWhere( + this.buildComparisonClause({ + dbType, + columnName: tieBreakerColumnName, + paramName: tieBreakerParamName, + operator, + }), + cursors, + ); + }), + ); + } + private toPagingResult(entities: Entity[]): PagingResult { return { data: entities, From 0e1098e899a1088b5fd3258a5edf3bf238fee56c Mon Sep 17 00:00:00 2001 From: Ravi Kiran Date: Mon, 9 Mar 2026 12:29:55 +0530 Subject: [PATCH 2/8] Remove some unnecessary changes --- .../server/api/src/app/helper/pagination/paginator.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/server/api/src/app/helper/pagination/paginator.ts b/packages/server/api/src/app/helper/pagination/paginator.ts index 903a168460..5f63aef7c5 100644 --- a/packages/server/api/src/app/helper/pagination/paginator.ts +++ b/packages/server/api/src/app/helper/pagination/paginator.ts @@ -170,10 +170,10 @@ export default class Paginator { const cursors: CursorParam = {}; const clonedBuilder = new SelectQueryBuilder(builder); - if (this.afterCursor !== null) { - Object.assign(cursors, this.decode(this.afterCursor)); - } else if (this.beforeCursor !== null) { - Object.assign(cursors, this.decode(this.beforeCursor)); + if (this.hasAfterCursor()) { + Object.assign(cursors, this.decode(this.afterCursor!)); + } else if (this.hasBeforeCursor()) { + Object.assign(cursors, this.decode(this.beforeCursor!)); } if (Object.keys(cursors).length > 0) { @@ -273,7 +273,7 @@ export default class Paginator { } const value = getValueByPath(entity, this.paginationColumnPath); - if (value === null || value === undefined) { + if (!value) { throw new Error( `Pagination column not found at path: ${this.paginationColumnPath}`, ); From 76010d0f737a4d706a48c8c6a335a6436bc037b6 Mon Sep 17 00:00:00 2001 From: Ravi Kiran Date: Mon, 9 Mar 2026 12:37:47 +0530 Subject: [PATCH 3/8] Use better names --- .../api/src/app/flows/flow/flow.service.ts | 2 +- .../app/helper/pagination/build-paginator.ts | 12 +-- .../src/app/helper/pagination/paginator.ts | 80 +++++++++---------- 3 files changed, 47 insertions(+), 47 deletions(-) diff --git a/packages/server/api/src/app/flows/flow/flow.service.ts b/packages/server/api/src/app/flows/flow/flow.service.ts index 4be7134e32..88c373034d 100644 --- a/packages/server/api/src/app/flows/flow/flow.service.ts +++ b/packages/server/api/src/app/flows/flow/flow.service.ts @@ -145,7 +145,7 @@ export const flowService = { columnName: 'fv.updated', columnType: 'timestamp with time zone', }, - customPaginationTieBreakerColumn: { + customPaginationSecondaryColumn: { columnPath: 'id', columnName: 'flow.id', columnType: 'string', diff --git a/packages/server/api/src/app/helper/pagination/build-paginator.ts b/packages/server/api/src/app/helper/pagination/build-paginator.ts index 9d3486b342..471278681a 100644 --- a/packages/server/api/src/app/helper/pagination/build-paginator.ts +++ b/packages/server/api/src/app/helper/pagination/build-paginator.ts @@ -17,7 +17,7 @@ export type PaginationOptions = { columnName: string; columnType?: string; }; - customPaginationTieBreakerColumn?: { + customPaginationSecondaryColumn?: { columnPath: string; columnName: string; columnType?: string; @@ -45,11 +45,11 @@ export function buildPaginator( ); } - if (options.customPaginationTieBreakerColumn) { - paginator.setPaginationTieBreakerColumn( - options.customPaginationTieBreakerColumn.columnPath, - options.customPaginationTieBreakerColumn.columnName, - options.customPaginationTieBreakerColumn.columnType, + if (options.customPaginationSecondaryColumn) { + paginator.setPaginationSecondaryColumn( + options.customPaginationSecondaryColumn.columnPath, + options.customPaginationSecondaryColumn.columnName, + options.customPaginationSecondaryColumn.columnType, ); } diff --git a/packages/server/api/src/app/helper/pagination/paginator.ts b/packages/server/api/src/app/helper/pagination/paginator.ts index 5f63aef7c5..d1b262019f 100644 --- a/packages/server/api/src/app/helper/pagination/paginator.ts +++ b/packages/server/api/src/app/helper/pagination/paginator.ts @@ -34,13 +34,13 @@ export type PagingResult = { type CursorContext = { primaryColumnName: string; primaryParamName: string; - tieBreakerColumnName: string | null; - tieBreakerParamName: string | null; + secondaryColumnName: string | null; + secondaryParamName: string | null; }; const PAGINATION_KEY = 'created'; const CUSTOM_PAGINATION_KEY = 'custom_pagination'; -const CUSTOM_PAGINATION_TIE_BREAKER_KEY = 'custom_pagination_tie_breaker'; +const CUSTOM_PAGINATION_SECONDARY_KEY = 'custom_pagination_tie_breaker'; const DEFAULT_TIMESTAMP_TYPE = 'timestamp with time zone'; export default class Paginator { @@ -64,11 +64,11 @@ export default class Paginator { private paginationColumnType: string | null = null; - private paginationTieBreakerColumnPath: string | null = null; + private paginationSecondaryColumnPath: string | null = null; - private paginationTieBreakerColumnName: string | null = null; + private paginationSecondaryColumnName: string | null = null; - private paginationTieBreakerColumnType: string | null = null; + private paginationSecondaryColumnType: string | null = null; public constructor(private readonly entity: EntitySchema) {} @@ -86,14 +86,14 @@ export default class Paginator { this.alias = alias; } - public setPaginationTieBreakerColumn( + public setPaginationSecondaryColumn( columnPath: string, columnName: string, columnType = 'string', ): void { - this.paginationTieBreakerColumnPath = columnPath; - this.paginationTieBreakerColumnName = columnName; - this.paginationTieBreakerColumnType = columnType; + this.paginationSecondaryColumnPath = columnPath; + this.paginationSecondaryColumnName = columnName; + this.paginationSecondaryColumnType = columnType; } public setAfterCursor(cursor: string): void { @@ -197,7 +197,7 @@ export default class Paginator { const operator = this.getOperator(); const context = this.resolveCursorContext(cursors); - if (context.tieBreakerColumnName && context.tieBreakerParamName) { + if (context.secondaryColumnName && context.secondaryParamName) { this.applyCompositeCursorFilter( where, cursors, @@ -244,8 +244,8 @@ export default class Paginator { orderByCondition[`${this.alias}.${PAGINATION_KEY}`] = order; } - if (this.paginationTieBreakerColumnName) { - orderByCondition[this.paginationTieBreakerColumnName] = order; + if (this.paginationSecondaryColumnName) { + orderByCondition[this.paginationSecondaryColumnName] = order; } return orderByCondition; @@ -286,24 +286,24 @@ export default class Paginator { const payload = [`${CUSTOM_PAGINATION_KEY}:${encodedValue}`]; if ( - this.paginationTieBreakerColumnPath && - this.paginationTieBreakerColumnName + this.paginationSecondaryColumnPath && + this.paginationSecondaryColumnName ) { - const tieBreakerValue = getValueByPath( + const secondaryValue = getValueByPath( entity, - this.paginationTieBreakerColumnPath, + this.paginationSecondaryColumnPath, ); - if (tieBreakerValue === null || tieBreakerValue === undefined) { + if (secondaryValue === null || secondaryValue === undefined) { throw new Error( - `Pagination tie breaker column not found at path: ${this.paginationTieBreakerColumnPath}`, + `Pagination secondary column not found at path: ${this.paginationSecondaryColumnPath}`, ); } - const encodedTieBreakerValue = encodeByType( - this.paginationTieBreakerColumnType || 'string', - tieBreakerValue, + const encodedSecondaryValue = encodeByType( + this.paginationSecondaryColumnType || 'string', + secondaryValue, ); payload.push( - `${CUSTOM_PAGINATION_TIE_BREAKER_KEY}:${encodedTieBreakerValue}`, + `${CUSTOM_PAGINATION_SECONDARY_KEY}:${encodedSecondaryValue}`, ); } @@ -327,8 +327,8 @@ export default class Paginator { if (key === CUSTOM_PAGINATION_KEY) { return this.paginationColumnType || DEFAULT_TIMESTAMP_TYPE; } - if (key === CUSTOM_PAGINATION_TIE_BREAKER_KEY) { - return this.paginationTieBreakerColumnType || 'string'; + if (key === CUSTOM_PAGINATION_SECONDARY_KEY) { + return this.paginationSecondaryColumnType || 'string'; } const col = this.entity.options.columns[key]; @@ -407,24 +407,24 @@ export default class Paginator { ? CUSTOM_PAGINATION_KEY : PAGINATION_KEY; - const hasCustomTieBreakerCursor = - this.paginationTieBreakerColumnName !== null && - cursors[CUSTOM_PAGINATION_TIE_BREAKER_KEY] !== undefined; + const hasCustomSecondaryCursor = + this.paginationSecondaryColumnName !== null && + cursors[CUSTOM_PAGINATION_SECONDARY_KEY] !== undefined; - if (hasCustomPaginationCursor && hasCustomTieBreakerCursor) { + if (hasCustomPaginationCursor && hasCustomSecondaryCursor) { return { primaryColumnName, primaryParamName, - tieBreakerColumnName: this.paginationTieBreakerColumnName, - tieBreakerParamName: CUSTOM_PAGINATION_TIE_BREAKER_KEY, + secondaryColumnName: this.paginationSecondaryColumnName, + secondaryParamName: CUSTOM_PAGINATION_SECONDARY_KEY, }; } return { primaryColumnName, primaryParamName, - tieBreakerColumnName: null, - tieBreakerParamName: null, + secondaryColumnName: null, + secondaryParamName: null, }; } @@ -456,11 +456,11 @@ export default class Paginator { const { primaryColumnName, primaryParamName, - tieBreakerColumnName, - tieBreakerParamName, + secondaryColumnName, + secondaryParamName, } = context; - if (!tieBreakerColumnName || !tieBreakerParamName) { - throw new Error('Pagination tie breaker context is not configured'); + if (!secondaryColumnName || !secondaryParamName) { + throw new Error('Pagination secondary context is not configured'); } where.orWhere( @@ -473,7 +473,7 @@ export default class Paginator { cursors, ); - // Lexicographic cursor compare: primary equals, then compare tie-breaker. + // Lexicographic cursor compare: primary equals, then compare secondary key. where.orWhere( new Brackets((nestedWhere) => { nestedWhere.where( @@ -488,8 +488,8 @@ export default class Paginator { nestedWhere.andWhere( this.buildComparisonClause({ dbType, - columnName: tieBreakerColumnName, - paramName: tieBreakerParamName, + columnName: secondaryColumnName, + paramName: secondaryParamName, operator, }), cursors, From 758ff11b4cad8d5b7038f96fb253b605020a219d Mon Sep 17 00:00:00 2001 From: Ravi Kiran Date: Mon, 9 Mar 2026 13:30:27 +0530 Subject: [PATCH 4/8] Add integration test cases --- .../pagination/paginator.integration.test.ts | 103 ++++++++++++++++++ 1 file changed, 103 insertions(+) diff --git a/packages/server/api/test/integration/helper/pagination/paginator.integration.test.ts b/packages/server/api/test/integration/helper/pagination/paginator.integration.test.ts index bd292934a7..07b364fd57 100644 --- a/packages/server/api/test/integration/helper/pagination/paginator.integration.test.ts +++ b/packages/server/api/test/integration/helper/pagination/paginator.integration.test.ts @@ -310,6 +310,109 @@ describe('Paginator Integration Tests', () => { }); }); + describe('Composite Pagination with Secondary Column', () => { + test('should paginate same-updated rows without skipping (limit 10, >20 rows)', async () => { + const sharedCreated = '2025-01-01 08:51:00.123'; + const totalRows = 25; + const rows = Array.from({ length: totalRows }, (_, index) => ({ + id: `run-${String(index + 1).padStart(3, '0')}`, + created: sharedCreated, + projectId: 'proj1', + status: 'SUCCEEDED', + })); + + for (const row of rows) { + await dataSource + .createQueryBuilder() + .insert() + .into('test_flow_runs') + .values(row) + .execute(); + } + + const baseQuery = () => + dataSource + .createQueryBuilder(TestFlowRunEntity, 'fr') + .where('fr.projectId = :projectId', { projectId: 'proj1' }); + + const expectedSortedIds = rows + .map((row) => row.id) + .sort((a, b) => b.localeCompare(a)); + + const page1Paginator = new Paginator(TestFlowRunEntity); + page1Paginator.setAlias('fr'); + page1Paginator.setOrder(Order.DESC); + page1Paginator.setLimit(10); + page1Paginator.setPaginationColumn('created', 'fr.created', 'datetime'); + page1Paginator.setPaginationSecondaryColumn('id', 'fr.id', 'string'); + const page1 = await page1Paginator.paginate(baseQuery()); + + const page2Paginator = new Paginator(TestFlowRunEntity); + page2Paginator.setAlias('fr'); + page2Paginator.setOrder(Order.DESC); + page2Paginator.setLimit(10); + page2Paginator.setPaginationColumn('created', 'fr.created', 'datetime'); + page2Paginator.setPaginationSecondaryColumn('id', 'fr.id', 'string'); + page2Paginator.setAfterCursor(page1.cursor.afterCursor!); + const page2 = await page2Paginator.paginate(baseQuery()); + + const combinedIds = [...page1.data, ...page2.data].map((row) => row.id); + + expect(page1.data).toHaveLength(10); + expect(page2.data).toHaveLength(10); + expect(combinedIds).toEqual(expectedSortedIds.slice(0, 20)); + }); + + test('should not duplicate rows across consecutive pages', async () => { + const sharedCreated = '2025-01-01 08:51:00.456'; + const rows = Array.from({ length: 15 }, (_, index) => ({ + id: `row-${String(index + 1).padStart(3, '0')}`, + created: sharedCreated, + projectId: 'proj1', + status: 'RUNNING', + })); + + for (const row of rows) { + await dataSource + .createQueryBuilder() + .insert() + .into('test_flow_runs') + .values(row) + .execute(); + } + + const baseQuery = () => + dataSource + .createQueryBuilder(TestFlowRunEntity, 'fr') + .where('fr.projectId = :projectId', { projectId: 'proj1' }); + + const page1Paginator = new Paginator(TestFlowRunEntity); + page1Paginator.setAlias('fr'); + page1Paginator.setOrder(Order.DESC); + page1Paginator.setLimit(10); + page1Paginator.setPaginationColumn('created', 'fr.created', 'datetime'); + page1Paginator.setPaginationSecondaryColumn('id', 'fr.id', 'string'); + const page1 = await page1Paginator.paginate(baseQuery()); + + const page2Paginator = new Paginator(TestFlowRunEntity); + page2Paginator.setAlias('fr'); + page2Paginator.setOrder(Order.DESC); + page2Paginator.setLimit(10); + page2Paginator.setPaginationColumn('created', 'fr.created', 'datetime'); + page2Paginator.setPaginationSecondaryColumn('id', 'fr.id', 'string'); + page2Paginator.setAfterCursor(page1.cursor.afterCursor!); + const page2 = await page2Paginator.paginate(baseQuery()); + + const page1Ids = page1.data.map((row) => row.id); + const page2Ids = page2.data.map((row) => row.id); + const duplicateIds = page1Ids.filter((id) => page2Ids.includes(id)); + + expect(page1.data).toHaveLength(10); + expect(page2.data).toHaveLength(5); + expect(duplicateIds).toHaveLength(0); + }); + }); + describe('Edge Cases', () => { describe('refetch when backward result is shorter than limit', () => { test.each([3, 4])( From 7bd00f40ace1a27134e8ca44697e03ba6b0e466a Mon Sep 17 00:00:00 2001 From: Ravi Kiran Date: Mon, 9 Mar 2026 14:55:43 +0530 Subject: [PATCH 5/8] Guard secondary column to be used only with primary column --- .../app/helper/pagination/build-paginator.ts | 38 +++++++++++++------ .../src/app/helper/pagination/paginator.ts | 2 +- 2 files changed, 28 insertions(+), 12 deletions(-) diff --git a/packages/server/api/src/app/helper/pagination/build-paginator.ts b/packages/server/api/src/app/helper/pagination/build-paginator.ts index 471278681a..cf9de8df59 100644 --- a/packages/server/api/src/app/helper/pagination/build-paginator.ts +++ b/packages/server/api/src/app/helper/pagination/build-paginator.ts @@ -8,21 +8,28 @@ export type PagingQuery = { order?: Order | 'ASC' | 'DESC'; }; +type CustomPaginationColumnOptions = { + columnPath: string; + columnName: string; + columnType?: string; +}; + +// Secondary custom pagination is only valid when primary custom pagination is configured. +type CustomPaginationColumns = + | { + customPaginationColumn?: undefined; + customPaginationSecondaryColumn?: undefined; + } + | { + customPaginationColumn: CustomPaginationColumnOptions; + customPaginationSecondaryColumn?: CustomPaginationColumnOptions; + }; + export type PaginationOptions = { entity: EntitySchema; alias?: string; query?: PagingQuery; - customPaginationColumn?: { - columnPath: string; - columnName: string; - columnType?: string; - }; - customPaginationSecondaryColumn?: { - columnPath: string; - columnName: string; - columnType?: string; - }; -}; +} & CustomPaginationColumns; export function buildPaginator( options: PaginationOptions, @@ -37,6 +44,15 @@ export function buildPaginator( paginator.setAlias(alias); + if ( + options.customPaginationSecondaryColumn && + !options.customPaginationColumn + ) { + throw new Error( + 'customPaginationSecondaryColumn requires customPaginationColumn', + ); + } + if (options.customPaginationColumn) { paginator.setPaginationColumn( options.customPaginationColumn.columnPath, diff --git a/packages/server/api/src/app/helper/pagination/paginator.ts b/packages/server/api/src/app/helper/pagination/paginator.ts index d1b262019f..3e2999b738 100644 --- a/packages/server/api/src/app/helper/pagination/paginator.ts +++ b/packages/server/api/src/app/helper/pagination/paginator.ts @@ -244,7 +244,7 @@ export default class Paginator { orderByCondition[`${this.alias}.${PAGINATION_KEY}`] = order; } - if (this.paginationSecondaryColumnName) { + if (this.paginationColumnName && this.paginationSecondaryColumnName) { orderByCondition[this.paginationSecondaryColumnName] = order; } From 4e1d6f307db06116df674f49c8d5a4ef85589f59 Mon Sep 17 00:00:00 2001 From: Ravi Kiran Date: Mon, 9 Mar 2026 15:01:40 +0530 Subject: [PATCH 6/8] Fix sonar comments --- .../pagination/paginator.integration.test.ts | 27 ++++++++----------- 1 file changed, 11 insertions(+), 16 deletions(-) diff --git a/packages/server/api/test/integration/helper/pagination/paginator.integration.test.ts b/packages/server/api/test/integration/helper/pagination/paginator.integration.test.ts index 07b364fd57..b4fd8d5786 100644 --- a/packages/server/api/test/integration/helper/pagination/paginator.integration.test.ts +++ b/packages/server/api/test/integration/helper/pagination/paginator.integration.test.ts @@ -311,6 +311,11 @@ describe('Paginator Integration Tests', () => { }); describe('Composite Pagination with Secondary Column', () => { + const buildProjectQuery = () => + dataSource + .createQueryBuilder(TestFlowRunEntity, 'fr') + .where('fr.projectId = :projectId', { projectId: 'proj1' }); + test('should paginate same-updated rows without skipping (limit 10, >20 rows)', async () => { const sharedCreated = '2025-01-01 08:51:00.123'; const totalRows = 25; @@ -330,11 +335,6 @@ describe('Paginator Integration Tests', () => { .execute(); } - const baseQuery = () => - dataSource - .createQueryBuilder(TestFlowRunEntity, 'fr') - .where('fr.projectId = :projectId', { projectId: 'proj1' }); - const expectedSortedIds = rows .map((row) => row.id) .sort((a, b) => b.localeCompare(a)); @@ -345,7 +345,7 @@ describe('Paginator Integration Tests', () => { page1Paginator.setLimit(10); page1Paginator.setPaginationColumn('created', 'fr.created', 'datetime'); page1Paginator.setPaginationSecondaryColumn('id', 'fr.id', 'string'); - const page1 = await page1Paginator.paginate(baseQuery()); + const page1 = await page1Paginator.paginate(buildProjectQuery()); const page2Paginator = new Paginator(TestFlowRunEntity); page2Paginator.setAlias('fr'); @@ -354,7 +354,7 @@ describe('Paginator Integration Tests', () => { page2Paginator.setPaginationColumn('created', 'fr.created', 'datetime'); page2Paginator.setPaginationSecondaryColumn('id', 'fr.id', 'string'); page2Paginator.setAfterCursor(page1.cursor.afterCursor!); - const page2 = await page2Paginator.paginate(baseQuery()); + const page2 = await page2Paginator.paginate(buildProjectQuery()); const combinedIds = [...page1.data, ...page2.data].map((row) => row.id); @@ -381,18 +381,13 @@ describe('Paginator Integration Tests', () => { .execute(); } - const baseQuery = () => - dataSource - .createQueryBuilder(TestFlowRunEntity, 'fr') - .where('fr.projectId = :projectId', { projectId: 'proj1' }); - const page1Paginator = new Paginator(TestFlowRunEntity); page1Paginator.setAlias('fr'); page1Paginator.setOrder(Order.DESC); page1Paginator.setLimit(10); page1Paginator.setPaginationColumn('created', 'fr.created', 'datetime'); page1Paginator.setPaginationSecondaryColumn('id', 'fr.id', 'string'); - const page1 = await page1Paginator.paginate(baseQuery()); + const page1 = await page1Paginator.paginate(buildProjectQuery()); const page2Paginator = new Paginator(TestFlowRunEntity); page2Paginator.setAlias('fr'); @@ -401,11 +396,11 @@ describe('Paginator Integration Tests', () => { page2Paginator.setPaginationColumn('created', 'fr.created', 'datetime'); page2Paginator.setPaginationSecondaryColumn('id', 'fr.id', 'string'); page2Paginator.setAfterCursor(page1.cursor.afterCursor!); - const page2 = await page2Paginator.paginate(baseQuery()); + const page2 = await page2Paginator.paginate(buildProjectQuery()); const page1Ids = page1.data.map((row) => row.id); - const page2Ids = page2.data.map((row) => row.id); - const duplicateIds = page1Ids.filter((id) => page2Ids.includes(id)); + const page2Ids = new Set(page2.data.map((row) => row.id)); + const duplicateIds = page1Ids.filter((id) => page2Ids.has(id)); expect(page1.data).toHaveLength(10); expect(page2.data).toHaveLength(5); From 8fc7a8cb0ad61bb7acbae630a2ae891ce1dd7407 Mon Sep 17 00:00:00 2001 From: Ravi Kiran Date: Mon, 9 Mar 2026 15:08:20 +0530 Subject: [PATCH 7/8] Fix sonar --- .../server/api/src/app/helper/pagination/build-paginator.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/server/api/src/app/helper/pagination/build-paginator.ts b/packages/server/api/src/app/helper/pagination/build-paginator.ts index cf9de8df59..da8d9fe7f0 100644 --- a/packages/server/api/src/app/helper/pagination/build-paginator.ts +++ b/packages/server/api/src/app/helper/pagination/build-paginator.ts @@ -17,8 +17,8 @@ type CustomPaginationColumnOptions = { // Secondary custom pagination is only valid when primary custom pagination is configured. type CustomPaginationColumns = | { - customPaginationColumn?: undefined; - customPaginationSecondaryColumn?: undefined; + customPaginationColumn?: never; + customPaginationSecondaryColumn?: never; } | { customPaginationColumn: CustomPaginationColumnOptions; From 38489e8d85a209ee23591da2a250998abd757948 Mon Sep 17 00:00:00 2001 From: Ravi Kiran Date: Mon, 9 Mar 2026 15:13:41 +0530 Subject: [PATCH 8/8] Extract a func to be used in multiple test cases --- .../pagination/paginator.integration.test.ts | 63 ++++++------------- 1 file changed, 18 insertions(+), 45 deletions(-) diff --git a/packages/server/api/test/integration/helper/pagination/paginator.integration.test.ts b/packages/server/api/test/integration/helper/pagination/paginator.integration.test.ts index b4fd8d5786..eea727759e 100644 --- a/packages/server/api/test/integration/helper/pagination/paginator.integration.test.ts +++ b/packages/server/api/test/integration/helper/pagination/paginator.integration.test.ts @@ -102,6 +102,10 @@ const FOUR_RUNS_TEST_DATA = [ describe('Paginator Integration Tests', () => { let dataSource: DataSource; + const buildFlowRunsQuery = (projectId = 'proj1') => + dataSource + .createQueryBuilder(TestFlowRunEntity, 'fr') + .where('fr.projectId = :projectId', { projectId }); beforeAll(async () => { dataSource = new DataSource({ @@ -174,9 +178,7 @@ describe('Paginator Integration Tests', () => { paginator.setOrder(Order.DESC); paginator.setLimit(2); - const query = dataSource - .createQueryBuilder(TestFlowRunEntity, 'fr') - .where('fr.projectId = :projectId', { projectId: 'proj1' }); + const query = buildFlowRunsQuery(); const result = await paginator.paginate(query); @@ -229,9 +231,7 @@ describe('Paginator Integration Tests', () => { paginator.setOrder(Order.DESC); paginator.setLimit(2); - let query = dataSource - .createQueryBuilder(TestFlowRunEntity, 'fr') - .where('fr.projectId = :projectId', { projectId: 'proj1' }); + let query = buildFlowRunsQuery(); const firstPage = await paginator.paginate(query); @@ -241,9 +241,7 @@ describe('Paginator Integration Tests', () => { paginator2.setLimit(2); paginator2.setAfterCursor(firstPage.cursor.afterCursor!); - query = dataSource - .createQueryBuilder(TestFlowRunEntity, 'fr') - .where('fr.projectId = :projectId', { projectId: 'proj1' }); + query = buildFlowRunsQuery(); const secondPage = await paginator2.paginate(query); @@ -253,9 +251,7 @@ describe('Paginator Integration Tests', () => { paginator3.setLimit(2); paginator3.setBeforeCursor(secondPage.cursor.beforeCursor!); - query = dataSource - .createQueryBuilder(TestFlowRunEntity, 'fr') - .where('fr.projectId = :projectId', { projectId: 'proj1' }); + query = buildFlowRunsQuery(); const backwardPage = await paginator3.paginate(query); @@ -298,9 +294,7 @@ describe('Paginator Integration Tests', () => { paginator.setPaginationColumn('created', 'fr.created', 'datetime'); - const query = dataSource - .createQueryBuilder(TestFlowRunEntity, 'fr') - .where('fr.projectId = :projectId', { projectId: 'proj1' }); + const query = buildFlowRunsQuery(); const result = await paginator.paginate(query); @@ -311,10 +305,7 @@ describe('Paginator Integration Tests', () => { }); describe('Composite Pagination with Secondary Column', () => { - const buildProjectQuery = () => - dataSource - .createQueryBuilder(TestFlowRunEntity, 'fr') - .where('fr.projectId = :projectId', { projectId: 'proj1' }); + const buildProjectQuery = () => buildFlowRunsQuery(); test('should paginate same-updated rows without skipping (limit 10, >20 rows)', async () => { const sharedCreated = '2025-01-01 08:51:00.123'; @@ -424,10 +415,7 @@ describe('Paginator Integration Tests', () => { .execute(); } - const queryBase = () => - dataSource - .createQueryBuilder(TestFlowRunEntity, 'fr') - .where('fr.projectId = :projectId', { projectId: 'proj1' }); + const queryBase = () => buildFlowRunsQuery(); const paginator1 = new Paginator(TestFlowRunEntity); paginator1.setAlias('fr'); @@ -474,10 +462,7 @@ describe('Paginator Integration Tests', () => { .execute(); } - const queryBase = () => - dataSource - .createQueryBuilder(TestFlowRunEntity, 'fr') - .where('fr.projectId = :projectId', { projectId: 'proj1' }); + const queryBase = () => buildFlowRunsQuery(); const paginator1 = new Paginator(TestFlowRunEntity); paginator1.setAlias('fr'); @@ -513,9 +498,7 @@ describe('Paginator Integration Tests', () => { paginator.setAlias('fr'); paginator.setLimit(10); - const query = dataSource - .createQueryBuilder(TestFlowRunEntity, 'fr') - .where('fr.projectId = :projectId', { projectId: 'nonexistent' }); + const query = buildFlowRunsQuery('nonexistent'); const result = await paginator.paginate(query); @@ -541,9 +524,7 @@ describe('Paginator Integration Tests', () => { paginator.setAlias('fr'); paginator.setLimit(10); - const query = dataSource - .createQueryBuilder(TestFlowRunEntity, 'fr') - .where('fr.projectId = :projectId', { projectId: 'proj1' }); + const query = buildFlowRunsQuery(); const result = await paginator.paginate(query); @@ -589,9 +570,7 @@ describe('Paginator Integration Tests', () => { paginator.setOrder(Order.DESC); paginator.setLimit(3); - const query = dataSource - .createQueryBuilder(TestFlowRunEntity, 'fr') - .where('fr.projectId = :projectId', { projectId: 'proj1' }); + const query = buildFlowRunsQuery(); const result = await paginator.paginate(query); @@ -644,9 +623,7 @@ describe('Paginator Integration Tests', () => { paginator1.setOrder(Order.DESC); paginator1.setLimit(2); - let query = dataSource - .createQueryBuilder(TestFlowRunEntity, 'fr') - .where('fr.projectId = :projectId', { projectId: 'proj1' }); + let query = buildFlowRunsQuery(); const page1 = await paginator1.paginate(query); @@ -656,9 +633,7 @@ describe('Paginator Integration Tests', () => { paginator2.setLimit(2); paginator2.setAfterCursor(page1.cursor.afterCursor!); - query = dataSource - .createQueryBuilder(TestFlowRunEntity, 'fr') - .where('fr.projectId = :projectId', { projectId: 'proj1' }); + query = buildFlowRunsQuery(); const page2 = await paginator2.paginate(query); @@ -668,9 +643,7 @@ describe('Paginator Integration Tests', () => { paginator3.setLimit(2); paginator3.setBeforeCursor(page2.cursor.beforeCursor!); - query = dataSource - .createQueryBuilder(TestFlowRunEntity, 'fr') - .where('fr.projectId = :projectId', { projectId: 'proj1' }); + query = buildFlowRunsQuery(); const backPage = await paginator3.paginate(query);