From fed24d19600fab3a6820016ed621499272993507 Mon Sep 17 00:00:00 2001 From: Morfixx <63129378+nahSystemu@users.noreply.github.com> Date: Sun, 21 Sep 2025 00:17:41 +0300 Subject: [PATCH] fix: bulkCreate to stop imports with duplicate indices (#190) * fix(db): normalize/compact indices to prevent import duplicates - bulkCreate now assigns sequential indices per board (preserves order; ignores incoming indices). - Auto-compact indices after create/insert if duplicates exist, keeping active lists unique and gap-free. - Prevents reorder errors caused by duplicate indices from Trello imports. * feat(db): normalize/compact indices with fail-safe duplicate recheck --- packages/db/src/repository/list.repo.ts | 151 ++++++++++++++++++++++-- 1 file changed, 144 insertions(+), 7 deletions(-) diff --git a/packages/db/src/repository/list.repo.ts b/packages/db/src/repository/list.repo.ts index 36e449be..ad553bcb 100644 --- a/packages/db/src/repository/list.repo.ts +++ b/packages/db/src/repository/list.repo.ts @@ -59,9 +59,32 @@ export const create = async ( .having(gt(countExpr, 1)); if (duplicateIndices.length > 0) { - throw new Error( - `Duplicate indices found after reordering in board ${result.boardId}`, - ); + // Compact indices to sequential values (0..n-1) to resolve duplicates while preserving order + await tx.execute(sql` + WITH ordered AS ( + SELECT id, ROW_NUMBER() OVER (ORDER BY "index", id) - 1 AS new_index + FROM "list" + WHERE "boardId" = ${result.boardId} AND "deletedAt" IS NULL + ) + UPDATE "list" l + SET "index" = o.new_index + FROM ordered o + WHERE l.id = o.id; + `); + + // Last resort: verify fix; if duplicates persist (e.g., due to race conditions), rollback + const postFixDupes = await tx + .select({ index: lists.index, count: countExpr }) + .from(lists) + .where(and(eq(lists.boardId, result.boardId), isNull(lists.deletedAt))) + .groupBy(lists.index) + .having(gt(countExpr, 1)); + + if (postFixDupes.length > 0) { + throw new Error( + `Invariant violation: duplicate indices remain after compaction in board ${result.boardId}`, + ); + } } return result; @@ -79,7 +102,98 @@ export const bulkCreate = async ( importId?: number; }[], ) => { - return db.insert(lists).values(listInput).returning(); + if (listInput.length === 0) return []; + + return db.transaction(async (tx) => { + // Group incoming rows by board to compute safe, sequential indices per board + const byBoard = new Map(); + for (const item of listInput) { + const arr = byBoard.get(item.boardId) ?? []; + arr.push(item); + byBoard.set(item.boardId, arr); + } + + const allValuesToInsert: { + publicId: string; + name: string; + createdBy: string; + boardId: number; + index: number; + importId?: number; + }[] = []; + + // For each board, append incoming lists after the current max index, preserving their relative order + for (const [boardId, items] of byBoard.entries()) { + // Find current max index for non-deleted lists in this board + const last = await tx.query.lists.findFirst({ + columns: { index: true }, + where: and(eq(lists.boardId, boardId), isNull(lists.deletedAt)), + orderBy: [desc(lists.index)], + }); + + let nextIndex = last ? last.index + 1 : 0; + + // Sort incoming by their provided index to preserve Trello order, then reassign sequential indices + const sorted = [...items].sort((a, b) => a.index - b.index); + for (const it of sorted) { + allValuesToInsert.push({ + publicId: it.publicId, + name: it.name, + createdBy: it.createdBy, + boardId: it.boardId, + index: nextIndex++, + importId: it.importId, + }); + } + } + + // Insert all rows in one go + const inserted = await tx + .insert(lists) + .values(allValuesToInsert) + .returning(); + + // Post-insert check: if duplicates exist, compact indices per board instead of failing + const countExpr = sql`COUNT(*)`.mapWith(Number); + for (const boardId of byBoard.keys()) { + const duplicateIndices = await tx + .select({ index: lists.index, count: countExpr }) + .from(lists) + .where(and(eq(lists.boardId, boardId), isNull(lists.deletedAt))) + .groupBy(lists.index) + .having(gt(countExpr, 1)); + + if (duplicateIndices.length > 0) { + await tx.execute(sql` + WITH ordered AS ( + SELECT id, ROW_NUMBER() OVER (ORDER BY "index", id) - 1 AS new_index + FROM "list" + WHERE "boardId" = ${boardId} AND "deletedAt" IS NULL + ) + UPDATE "list" l + SET "index" = o.new_index + FROM ordered o + WHERE l.id = o.id; + `); + + // Last resort: verify fix; if duplicates persist (e.g., due to race conditions), rollback + const postFixDupes = await tx + .select({ index: lists.index, count: countExpr }) + .from(lists) + .where(and(eq(lists.boardId, boardId), isNull(lists.deletedAt))) + .groupBy(lists.index) + .having(gt(countExpr, 1)); + + if (postFixDupes.length > 0) { + throw new Error( + `Invariant violation: duplicate indices remain after compaction in board ${boardId}`, + ); + } + } + } + + return inserted; + }); }; export const getByPublicId = async (db: dbClient, listPublicId: string) => { @@ -180,9 +294,32 @@ export const reorder = async ( .having(gt(countExpr, 1)); if (duplicateIndices.length > 0) { - throw new Error( - `Duplicate indices found after reordering in board ${list.boardId}`, - ); + // Attempt to auto-heal by compacting indices to sequential values (0..n-1) while preserving order + await tx.execute(sql` + WITH ordered AS ( + SELECT id, ROW_NUMBER() OVER (ORDER BY "index", id) - 1 AS new_index + FROM "list" + WHERE "boardId" = ${list.boardId} AND "deletedAt" IS NULL + ) + UPDATE "list" l + SET "index" = o.new_index + FROM ordered o + WHERE l.id = o.id; + `); + + // Last resort verification: if duplicates persist, rollback + const postFixDupes = await tx + .select({ index: lists.index, count: countExpr }) + .from(lists) + .where(and(eq(lists.boardId, list.boardId), isNull(lists.deletedAt))) + .groupBy(lists.index) + .having(gt(countExpr, 1)); + + if (postFixDupes.length > 0) { + throw new Error( + `Invariant violation: duplicate indices remain after compaction in board ${list.boardId}`, + ); + } } const updatedList = await tx.query.lists.findFirst({